diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 0777507aec45fc..2ba16a9b39c694 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -1900,7 +1900,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) { ssize_t imm1; ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); assert(insOptsScalableStandard(id->idInsOpt())); assert(isVectorRegister(id->idReg1())); // ddddd assert(isValidSimm<5>(imm1)); // iiiii @@ -3310,18 +3310,6 @@ static const char * const wRegNames[] = #include "register.h" }; - -static const char * const zRegNames[] = -{ - "z0", "z1", "z2", "z3", "z4", - "z5", "z6", "z7", "z8", "z9", - "z10", "z11", "z12", "z13", "z14", - "z15", "z16", "z17", "z18", "z19", - "z20", "z21", "z22", "z23", "z24", - "z25", "z26", "z27", "z28", "z29", - "z30", "z31" -}; - static const char * const vRegNames[] = { "v0", "v1", "v2", "v3", "v4", @@ -3365,34 +3353,6 @@ static const char * const bRegNames[] = "b30", "b31" }; -static const char * const pRegNames[] = -{ - "p0", "p1", "p2", "p3", "p4", - "p5", "p6", "p7", "p8", "p9", - "p10", "p11", "p12", "p13", "p14", - "p15" -}; - -static const char * const pnRegNames[] = -{ - "pn0", "pn1", "pn2", "pn3", "pn4", - "pn5", "pn6", "pn7", "pn8", "pn9", - "pn10", "pn11", "pn12", "pn13", "pn14", - "pn15" -}; - -static const char * const svePatternNames[] = -{ - "pow2", "vl1", "vl2", "vl3", - "vl4", "vl5", "vl6", "vl7", - "vl8", "vl16", "vl32", "vl64", - "vl128", "vl256", "invalid", "invalid", - "invalid", "invalid", "invalid", "invalid", - "invalid", "invalid", "invalid", "invalid", - "invalid", "invalid", "invalid", "invalid", - "invalid", "mul4", "mul3", "all" -}; - // clang-format on //------------------------------------------------------------------------ @@ -3436,7 +3396,7 @@ const char* emitter::emitRegName(regNumber reg, emitAttr size, bool varName) con } else if (size == EA_SCALABLE) { - rn = zRegNames[reg - REG_V0]; + rn = emitSveRegName(reg); } } @@ -3445,24 +3405,6 @@ const char* emitter::emitRegName(regNumber reg, emitAttr size, bool varName) con return rn; } -//------------------------------------------------------------------------ -// emitSveRegName: Returns a scalable vector register name. -// -// Arguments: -// reg - A SIMD and floating-point register. -// -// Return value: -// A string that represents a scalable vector register name. -// -const char* emitter::emitSveRegName(regNumber reg) -{ - assert((reg >= REG_V0) && (reg <= REG_V31)); - - int index = (int)reg - (int)REG_V0; - - return zRegNames[index]; -} - //------------------------------------------------------------------------ // emitVectorRegName: Returns a SIMD vector register name. // @@ -3481,25 +3423,6 @@ const char* emitter::emitVectorRegName(regNumber reg) return vRegNames[index]; } -//------------------------------------------------------------------------ -// emitPredicateRegName: Returns a predicate register name. -// -// Arguments: -// reg - A predicate register. -// -// Return value: -// A string that represents a predicate register name. -// -const char* emitter::emitPredicateRegName(regNumber reg, PredicateType ptype) -{ - assert((reg >= REG_P0) && (reg <= REG_P15)); - - const int index = (int)reg - (int)REG_P0; - const bool usePnRegs = (ptype == PREDICATE_N) || (ptype == PREDICATE_N_SIZED); - - return usePnRegs ? pnRegNames[index] : pRegNames[index]; -} - /***************************************************************************** * * Returns the base encoding of the given CPU instruction. @@ -4398,7502 +4321,1332 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) return code; } -/***************************************************************************** - * - * Returns the specific encoding of the given CPU instruction and format - */ +// true if this 'imm' can be encoded as a input operand to a mov instruction +/*static*/ bool emitter::emitIns_valid_imm_for_mov(INT64 imm, emitAttr size) +{ + // Check for "MOV (wide immediate)". + if (canEncodeHalfwordImm(imm, size)) + return true; -emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) + // Next try the ones-complement form of 'halfword immediate' imm(i16,hw), + // namely "MOV (inverted wide immediate)". + ssize_t notOfImm = NOT_helper(imm, getBitWidth(size)); + if (canEncodeHalfwordImm(notOfImm, size)) + return true; + + // Finally try "MOV (bitmask immediate)" imm(N,r,s) + if (canEncodeBitMaskImm(imm, size)) + return true; + + return false; +} + +// true if this 'imm' can be encoded as a input operand to a vector movi instruction +/*static*/ bool emitter::emitIns_valid_imm_for_movi(INT64 imm, emitAttr elemsize) { - // clang-format off - const static code_t insCodes1[] = + if (elemsize == EA_8BYTE) { - #define INST1(id, nm, info, fmt, e1 ) e1, - #define INST2(id, nm, info, fmt, e1, e2 ) e1, - #define INST3(id, nm, info, fmt, e1, e2, e3 ) e1, - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e1, - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e1, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e1, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e1, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e1, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e1, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e1, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e1, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes2[] = + UINT64 uimm = imm; + while (uimm != 0) + { + INT64 loByte = uimm & 0xFF; + if ((loByte == 0) || (loByte == 0xFF)) + { + uimm >>= 8; + } + else + { + return false; + } + } + assert(uimm == 0); + return true; + } + else { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) e2, - #define INST3(id, nm, info, fmt, e1, e2, e3 ) e2, - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e2, - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e2, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e2, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e2, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e2, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e2, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e2, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e2, - #include "instrsarm64sve.h" - }; + // First try the standard 'byteShifted immediate' imm(i8,bySh) + if (canEncodeByteShiftedImm(imm, elemsize, true)) + return true; - const static code_t insCodes3[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) e3, - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e3, - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e3, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e3, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e3, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e3, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e3, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e3, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e3, - #include "instrsarm64sve.h" - }; + // Next try the ones-complement form of the 'immediate' imm(i8,bySh) + ssize_t notOfImm = NOT_helper(imm, getBitWidth(elemsize)); + if (canEncodeByteShiftedImm(notOfImm, elemsize, true)) + return true; + } + return false; +} - const static code_t insCodes4[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e4, - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e4, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e4, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e4, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e4, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e4, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e4, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e4, - #include "instrsarm64sve.h" - }; +// true if this 'imm' can be encoded as a input operand to a fmov instruction +/*static*/ bool emitter::emitIns_valid_imm_for_fmov(double immDbl) +{ + if (canEncodeFloatImm8(immDbl)) + return true; - const static code_t insCodes5[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e5, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e5, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e5, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e5, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e5, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e5, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e5, - #include "instrsarm64sve.h" - }; + return false; +} - const static code_t insCodes6[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e6, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e6, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e6, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e6, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e6, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e6, - #include "instrsarm64sve.h" - }; +// true if this 'imm' can be encoded as a input operand to an add instruction +/*static*/ bool emitter::emitIns_valid_imm_for_add(INT64 imm, emitAttr size) +{ + if (unsigned_abs(imm) <= 0x0fff) + return true; + else if (canEncodeWithShiftImmBy12(imm)) // Try the shifted by 12 encoding + return true; - const static code_t insCodes7[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e7, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e7, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e7, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e7, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e7, - #include "instrsarm64sve.h" - }; + return false; +} - const static code_t insCodes8[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e8, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e8, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e8, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e8, - #include "instrsarm64sve.h" - }; +// true if this 'imm' can be encoded as a input operand to an non-add/sub alu instruction +/*static*/ bool emitter::emitIns_valid_imm_for_cmp(INT64 imm, emitAttr size) +{ + return emitIns_valid_imm_for_add(imm, size); +} - const static code_t insCodes9[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e9, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e9, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e9, - #include "instrsarm64sve.h" - }; +// true if this 'imm' can be encoded as a input operand to an non-add/sub alu instruction +/*static*/ bool emitter::emitIns_valid_imm_for_alu(INT64 imm, emitAttr size) +{ + if (canEncodeBitMaskImm(imm, size)) + return true; - const static code_t insCodes10[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e10, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e10, - #include "instrsarm64sve.h" - }; + return false; +} - const static code_t insCodes11[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e11, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e11, - #include "instrsarm64sve.h" - }; +// true if this 'imm' can be encoded as the offset in an unscaled ldr/str instruction +/*static*/ bool emitter::emitIns_valid_imm_for_unscaled_ldst_offset(INT64 imm) +{ + return (imm >= -256) && (imm <= 255); +} - const static code_t insCodes12[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e12, - #include "instrsarm64sve.h" - }; +// true if this 'imm' can be encoded as the offset in a ldr/str instruction +/*static*/ bool emitter::emitIns_valid_imm_for_ldst_offset(INT64 imm, emitAttr attr) +{ + if (imm == 0) + return true; // Encodable using IF_LS_2A - const static code_t insCodes13[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e13, - #include "instrsarm64sve.h" - }; + if ((imm >= -256) && (imm <= 255)) + return true; // Encodable using IF_LS_2C (or possibly IF_LS_2B) - // clang-format on - const static insFormat formatEncode13A[13] = {IF_SVE_AU_3A, IF_SVE_BT_1A, IF_SVE_BV_2A, IF_SVE_BV_2A_J, - IF_SVE_BW_2A, IF_SVE_CB_2A, IF_SVE_CP_3A, IF_SVE_CQ_3A, - IF_SVE_CW_4A, IF_SVE_CZ_4A, IF_SVE_CZ_4A_K, IF_SVE_CZ_4A_L, - IF_SVE_EB_1A}; - const static insFormat formatEncode11A[11] = {IF_SVE_JD_4B, IF_SVE_JD_4C, IF_SVE_JI_3A_A, IF_SVE_JJ_4A, - IF_SVE_JJ_4A_B, IF_SVE_JJ_4A_C, IF_SVE_JJ_4A_D, IF_SVE_JJ_4B, - IF_SVE_JJ_4B_E, IF_SVE_JN_3B, IF_SVE_JN_3C}; - const static insFormat formatEncode9A[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, - IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, - IF_SVE_HX_3A_E, IF_SVE_IJ_3A_F, IF_SVE_IK_4A_G}; - const static insFormat formatEncode9B[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, - IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, - IF_SVE_HX_3A_E, IF_SVE_IJ_3A_G, IF_SVE_IK_4A_I}; - const static insFormat formatEncode9C[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, - IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, - IF_SVE_HX_3A_E, IF_SVE_IH_3A_F, IF_SVE_II_4A_H}; - const static insFormat formatEncode9D[9] = {IF_SVE_IH_3A, IF_SVE_IH_3A_A, IF_SVE_II_4A, - IF_SVE_II_4A_B, IF_SVE_IU_4A, IF_SVE_IU_4A_C, - IF_SVE_IU_4B, IF_SVE_IU_4B_D, IF_SVE_IV_3A}; - const static insFormat formatEncode9E[9] = {IF_SVE_JD_4A, IF_SVE_JI_3A_A, IF_SVE_JJ_4A, - IF_SVE_JJ_4A_B, IF_SVE_JJ_4A_C, IF_SVE_JJ_4A_D, - IF_SVE_JJ_4B, IF_SVE_JJ_4B_E, IF_SVE_JN_3A}; - const static insFormat formatEncode9F[9] = {IF_SVE_JD_4C, IF_SVE_JD_4C_A, IF_SVE_JJ_4A, - IF_SVE_JJ_4A_B, IF_SVE_JJ_4B, IF_SVE_JJ_4B_C, - IF_SVE_JL_3A, IF_SVE_JN_3C, IF_SVE_JN_3C_D}; - const static insFormat formatEncode8A[8] = {IF_SVE_CE_2A, IF_SVE_CE_2B, IF_SVE_CE_2C, IF_SVE_CE_2D, - IF_SVE_CF_2A, IF_SVE_CF_2B, IF_SVE_CF_2C, IF_SVE_CF_2D}; - const static insFormat formatEncode8B[8] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, IF_SVE_HW_4A_C, - IF_SVE_HW_4B, IF_SVE_HW_4B_D, IF_SVE_HX_3A_E, IF_SVE_IG_4A_F}; - const static insFormat formatEncode8C[8] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, IF_SVE_HW_4A_C, - IF_SVE_HW_4B, IF_SVE_HW_4B_D, IF_SVE_HX_3A_E, IF_SVE_IG_4A_G}; - const static insFormat formatEncode7A[7] = {IF_SVE_IJ_3A, IF_SVE_IK_4A, IF_SVE_IU_4A, IF_SVE_IU_4A_A, - IF_SVE_IU_4B, IF_SVE_IU_4B_B, IF_SVE_IV_3A}; - const static insFormat formatEncode6A[6] = {IF_SVE_AE_3A, IF_SVE_BD_3A, IF_SVE_EE_1A, - IF_SVE_FD_3A, IF_SVE_FD_3B, IF_SVE_FD_3C}; - const static insFormat formatEncode6B[6] = {IF_SVE_GY_3A, IF_SVE_GY_3B, IF_SVE_GY_3B_D, - IF_SVE_HA_3A, IF_SVE_HA_3A_E, IF_SVE_HA_3A_F}; - const static insFormat formatEncode6C[6] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, - IF_SVE_HX_3A_B, IF_SVE_IJ_3A_D, IF_SVE_IK_4A_F}; - const static insFormat formatEncode6D[6] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, - IF_SVE_HX_3A_B, IF_SVE_IJ_3A_E, IF_SVE_IK_4A_H}; - const static insFormat formatEncode6E[6] = {IF_SVE_HY_3A, IF_SVE_HY_3A_A, IF_SVE_HY_3B, - IF_SVE_HZ_2A_B, IF_SVE_IA_2A, IF_SVE_IB_3A}; - const static insFormat formatEncode6F[6] = {IF_SVE_IG_4A, IF_SVE_IU_4A, IF_SVE_IU_4A_A, - IF_SVE_IU_4B, IF_SVE_IU_4B_B, IF_SVE_IV_3A}; - const static insFormat formatEncode6G[6] = {IF_SVE_JD_4A, IF_SVE_JI_3A_A, IF_SVE_JK_4A, - IF_SVE_JK_4A_B, IF_SVE_JK_4B, IF_SVE_JN_3A}; - const static insFormat formatEncode5A[5] = {IF_SVE_AM_2A, IF_SVE_AN_3A, IF_SVE_AO_3A, IF_SVE_BF_2A, IF_SVE_BG_3A}; - const static insFormat formatEncode5B[5] = {IF_SVE_GX_3A, IF_SVE_GX_3B, IF_SVE_HK_3A, IF_SVE_HL_3A, IF_SVE_HM_2A}; - const static insFormat formatEncode5C[5] = {IF_SVE_EF_3A, IF_SVE_EG_3A, IF_SVE_EH_3A, IF_SVE_EY_3A, IF_SVE_EY_3B}; - const static insFormat formatEncode5D[5] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, IF_SVE_HX_3A_B, - IF_SVE_IG_4A_D}; - const static insFormat formatEncode5E[5] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, IF_SVE_HX_3A_B, - IF_SVE_IG_4A_E}; - const static insFormat formatEncode4A[4] = {IF_SVE_AA_3A, IF_SVE_AU_3A, IF_SVE_BS_1A, IF_SVE_CZ_4A}; - const static insFormat formatEncode4B[4] = {IF_SVE_BU_2A, IF_SVE_BV_2B, IF_SVE_EA_1A, IF_SVE_EB_1B}; - const static insFormat formatEncode4E[4] = {IF_SVE_BE_3A, IF_SVE_FI_3A, IF_SVE_FI_3B, IF_SVE_FI_3C}; - const static insFormat formatEncode4F[4] = {IF_SVE_EM_3A, IF_SVE_FK_3A, IF_SVE_FK_3B, IF_SVE_FK_3C}; - const static insFormat formatEncode4G[4] = {IF_SVE_AR_4A, IF_SVE_FF_3A, IF_SVE_FF_3B, IF_SVE_FF_3C}; - const static insFormat formatEncode4H[4] = {IF_SVE_GM_3A, IF_SVE_GN_3A, IF_SVE_GZ_3A, IF_SVE_HB_3A}; - const static insFormat formatEncode4I[4] = {IF_SVE_AX_1A, IF_SVE_AY_2A, IF_SVE_AZ_2A, IF_SVE_BA_3A}; - const static insFormat formatEncode4J[4] = {IF_SVE_BV_2A, IF_SVE_BV_2A_A, IF_SVE_CP_3A, IF_SVE_CQ_3A}; - const static insFormat formatEncode4K[4] = {IF_SVE_IF_4A, IF_SVE_IF_4A_A, IF_SVE_IM_3A, IF_SVE_IN_4A}; - const static insFormat formatEncode4L[4] = {IF_SVE_IZ_4A, IF_SVE_IZ_4A_A, IF_SVE_JB_4A, IF_SVE_JM_3A}; - const static insFormat formatEncode3A[3] = {IF_SVE_AB_3A, IF_SVE_AT_3A, IF_SVE_EC_1A}; - const static insFormat formatEncode3B[3] = {IF_SVE_BH_3A, IF_SVE_BH_3B, IF_SVE_BH_3B_A}; - const static insFormat formatEncode3C[3] = {IF_SVE_BW_2A, IF_SVE_CB_2A, IF_SVE_EB_1A}; - const static insFormat formatEncode3D[3] = {IF_SVE_BR_3A, IF_SVE_BR_3B, IF_SVE_CI_3A}; - const static insFormat formatEncode3E[3] = {IF_SVE_AT_3A, IF_SVE_EC_1A, IF_SVE_ET_3A}; - const static insFormat formatEncode3F[3] = {IF_SVE_GU_3A, IF_SVE_GU_3B, IF_SVE_HU_4A}; - const static insFormat formatEncode3G[3] = {IF_SVE_GH_3A, IF_SVE_GH_3B, IF_SVE_GH_3B_B}; - const static insFormat formatEncode3H[3] = {IF_SVE_HK_3A, IF_SVE_HL_3A, IF_SVE_HM_2A}; - const static insFormat formatEncode3I[3] = {IF_SVE_CM_3A, IF_SVE_CN_3A, IF_SVE_CO_3A}; - const static insFormat formatEncode3J[3] = {IF_SVE_CX_4A, IF_SVE_CX_4A_A, IF_SVE_CY_3A}; - const static insFormat formatEncode3K[3] = {IF_SVE_CX_4A, IF_SVE_CX_4A_A, IF_SVE_CY_3B}; - const static insFormat formatEncode3L[3] = {IF_SVE_DT_3A, IF_SVE_DX_3A, IF_SVE_DY_3A}; - const static insFormat formatEncode3M[3] = {IF_SVE_EJ_3A, IF_SVE_FA_3A, IF_SVE_FA_3B}; - const static insFormat formatEncode3N[3] = {IF_SVE_EK_3A, IF_SVE_FB_3A, IF_SVE_FB_3B}; - const static insFormat formatEncode3O[3] = {IF_SVE_EK_3A, IF_SVE_FC_3A, IF_SVE_FC_3B}; - const static insFormat formatEncode3P[3] = {IF_SVE_EL_3A, IF_SVE_FG_3A, IF_SVE_FG_3B}; - const static insFormat formatEncode3Q[3] = {IF_SVE_EO_3A, IF_SVE_FJ_3A, IF_SVE_FJ_3B}; - const static insFormat formatEncode3R[3] = {IF_SVE_FE_3A, IF_SVE_FE_3B, IF_SVE_FN_3A}; - const static insFormat formatEncode3S[3] = {IF_SVE_FH_3A, IF_SVE_FH_3B, IF_SVE_FN_3A}; - const static insFormat formatEncode3T[3] = {IF_SVE_GX_3C, IF_SVE_HK_3B, IF_SVE_HL_3B}; - const static insFormat formatEncode3U[3] = {IF_SVE_IM_3A, IF_SVE_IN_4A, IF_SVE_IX_4A}; - const static insFormat formatEncode3V[3] = {IF_SVE_JA_4A, IF_SVE_JB_4A, IF_SVE_JM_3A}; - const static insFormat formatEncode2AA[2] = {IF_SVE_ID_2A, IF_SVE_IE_2A}; - const static insFormat formatEncode2AB[2] = {IF_SVE_JG_2A, IF_SVE_JH_2A}; - const static insFormat formatEncode2AC[2] = {IF_SVE_AD_3A, IF_SVE_ED_1A}; - const static insFormat formatEncode2AD[2] = {IF_SVE_AB_3B, IF_SVE_AT_3B}; - const static insFormat formatEncode2AE[2] = {IF_SVE_CG_2A, IF_SVE_CJ_2A}; - const static insFormat formatEncode2AF[2] = {IF_SVE_AE_3A, IF_SVE_BD_3A}; - const static insFormat formatEncode2AG[2] = {IF_SVE_BS_1A, IF_SVE_CZ_4A}; - const static insFormat formatEncode2AH[2] = {IF_SVE_BQ_2A, IF_SVE_BQ_2B}; - const static insFormat formatEncode2AI[2] = {IF_SVE_AM_2A, IF_SVE_EU_3A}; - const static insFormat formatEncode2AJ[2] = {IF_SVE_HI_3A, IF_SVE_HT_4A}; - const static insFormat formatEncode2AK[2] = {IF_SVE_BZ_3A, IF_SVE_BZ_3A_A}; - const static insFormat formatEncode2AL[2] = {IF_SVE_GG_3A, IF_SVE_GG_3B}; - const static insFormat formatEncode2AM[2] = {IF_SVE_HL_3A, IF_SVE_HM_2A}; - const static insFormat formatEncode2AN[2] = {IF_SVE_EI_3A, IF_SVE_EZ_3A}; - const static insFormat formatEncode2AO[2] = {IF_SVE_GT_4A, IF_SVE_GV_3A}; - const static insFormat formatEncode2AP[2] = {IF_SVE_GY_3B, IF_SVE_HA_3A}; - const static insFormat formatEncode2AQ[2] = {IF_SVE_GO_3A, IF_SVE_HC_3A}; - const static insFormat formatEncode2AR[2] = {IF_SVE_AP_3A, IF_SVE_CZ_4A}; - const static insFormat formatEncode2AT[2] = {IF_SVE_AB_3A, IF_SVE_EC_1A}; - const static insFormat formatEncode2AU[2] = {IF_SVE_AH_3A, IF_SVE_BI_2A}; - const static insFormat formatEncode2AV[2] = {IF_SVE_BM_1A, IF_SVE_BN_1A}; - const static insFormat formatEncode2AW[2] = {IF_SVE_BO_1A, IF_SVE_BP_1A}; - const static insFormat formatEncode2AX[2] = {IF_SVE_CC_2A, IF_SVE_CD_2A}; - const static insFormat formatEncode2AY[2] = {IF_SVE_CR_3A, IF_SVE_CS_3A}; - const static insFormat formatEncode2AZ[2] = {IF_SVE_CV_3A, IF_SVE_CV_3B}; - const static insFormat formatEncode2BA[2] = {IF_SVE_CW_4A, IF_SVE_CZ_4A}; - const static insFormat formatEncode2BB[2] = {IF_SVE_CZ_4A, IF_SVE_CZ_4A_A}; - const static insFormat formatEncode2BC[2] = {IF_SVE_DE_1A, IF_SVE_DZ_1A}; - const static insFormat formatEncode2BD[2] = {IF_SVE_DG_2A, IF_SVE_DH_1A}; - const static insFormat formatEncode2BE[2] = {IF_SVE_DK_3A, IF_SVE_DL_2A}; - const static insFormat formatEncode2BF[2] = {IF_SVE_DM_2A, IF_SVE_DN_2A}; - const static insFormat formatEncode2BG[2] = {IF_SVE_DO_2A, IF_SVE_DP_2A}; - const static insFormat formatEncode2BH[2] = {IF_SVE_DW_2A, IF_SVE_DW_2B}; - const static insFormat formatEncode2BI[2] = {IF_SVE_FN_3A, IF_SVE_FN_3B}; - const static insFormat formatEncode2BJ[2] = {IF_SVE_GQ_3A, IF_SVE_HG_2A}; - const static insFormat formatEncode2BK[2] = {IF_SVE_GU_3C, IF_SVE_HU_4B}; - const static insFormat formatEncode2BL[2] = {IF_SVE_GZ_3A, IF_SVE_HB_3A}; - const static insFormat formatEncode2BM[2] = {IF_SVE_HK_3B, IF_SVE_HL_3B}; - const static insFormat formatEncode2BN[2] = {IF_SVE_IF_4A, IF_SVE_IF_4A_A}; - const static insFormat formatEncode2BO[2] = {IF_SVE_IO_3A, IF_SVE_IP_4A}; - const static insFormat formatEncode2BP[2] = {IF_SVE_IQ_3A, IF_SVE_IR_4A}; - const static insFormat formatEncode2BQ[2] = {IF_SVE_IS_3A, IF_SVE_IT_4A}; - const static insFormat formatEncode2BR[2] = {IF_SVE_JC_4A, IF_SVE_JO_3A}; - const static insFormat formatEncode2BS[2] = {IF_SVE_JE_3A, IF_SVE_JF_4A}; + if (imm < 0) + return false; // not encodable - code_t code = BAD_CODE; - insFormat insFmt = emitInsFormat(ins); - bool encoding_found = false; - int index = -1; + emitAttr size = EA_SIZE(attr); + unsigned scale = NaturalScale_helper(size); + ssize_t mask = size - 1; // the mask of low bits that must be zero to encode the immediate - switch (insFmt) + if (((imm & mask) == 0) && ((imm >> scale) < 0x1000)) + return true; // Encodable using IF_LS_2B + + return false; // not encodable +} + +// true if this 'imm' can be encoded as a input operand to a ccmp instruction +/*static*/ bool emitter::emitIns_valid_imm_for_ccmp(INT64 imm) +{ + return ((imm & 0x01f) == imm); +} + +// true if 'imm' can be encoded as an offset in a ldp/stp instruction +/*static*/ bool emitter::canEncodeLoadOrStorePairOffset(INT64 imm, emitAttr attr) +{ + assert((attr == EA_4BYTE) || (attr == EA_8BYTE) || (attr == EA_16BYTE)); + const int size = EA_SIZE_IN_BYTES(attr); + return (imm % size == 0) && (imm >= -64 * size) && (imm < 64 * size); +} + +/************************************************************************ + * + * A helper method to return the natural scale for an EA 'size' + */ + +/*static*/ unsigned emitter::NaturalScale_helper(emitAttr size) +{ + assert(size == EA_1BYTE || size == EA_2BYTE || size == EA_4BYTE || size == EA_8BYTE || size == EA_16BYTE); + return BitOperations::Log2((unsigned)size); +} + +/************************************************************************ + * + * A helper method to perform a Rotate-Right shift operation + * the source is 'value' and it is rotated right by 'sh' bits + * 'value' is considered to be a fixed size 'width' set of bits. + * + * Example + * value is '00001111', sh is 2 and width is 8 + * result is '11000011' + */ + +/*static*/ UINT64 emitter::ROR_helper(UINT64 value, unsigned sh, unsigned width) +{ + assert(width <= 64); + // Check that 'value' fits in 'width' bits + assert((width == 64) || (value < (1ULL << width))); + // We don't support shifts >= width + assert(sh < width); + + UINT64 result; + + unsigned rsh = sh; + unsigned lsh = width - rsh; + + result = (value >> rsh); + result |= (value << lsh); + + if (width < 64) { - case IF_SVE_13A: - for (index = 0; index < 13; index++) - { - if (fmt == formatEncode13A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_11A: - for (index = 0; index < 11; index++) - { - if (fmt == formatEncode11A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9A: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9B: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9C: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9D: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9D[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9E: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9F: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9F[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_8A: - for (index = 0; index < 8; index++) - { - if (fmt == formatEncode8A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_8B: - for (index = 0; index < 8; index++) - { - if (fmt == formatEncode8B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_8C: - for (index = 0; index < 8; index++) - { - if (fmt == formatEncode8C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_7A: - for (index = 0; index < 7; index++) - { - if (fmt == formatEncode7A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6A: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6B: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6C: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6D: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6D[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6E: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6F: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6F[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6G: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6G[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5A: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5B: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5C: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5D: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5D[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5E: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4A: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4B: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4E: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4F: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4F[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4G: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4G[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4H: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4H[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4I: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4I[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4J: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4J[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4K: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4K[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4L: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4L[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3A: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3B: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3C: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3D: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3D[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3E: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3F: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3F[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3G: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3G[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3H: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3H[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3I: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3I[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3J: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3J[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3K: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3K[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3L: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3L[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3M: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3M[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3N: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3N[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3O: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3O[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3P: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3P[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3Q: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3Q[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3R: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3R[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3S: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3S[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3T: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3T[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3U: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3U[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3V: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3V[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AA: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AA[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AB: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AB[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AC: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AC[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AD: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AD[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AE: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AE[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AF: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AF[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AG: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AG[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AH: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AH[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AI: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AI[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AJ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AJ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AK: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AK[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AL: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AL[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AM: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AM[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AN: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AN[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AO: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AO[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AP: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AP[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AQ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AQ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AR: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AR[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AT: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AT[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AU: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AU[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AV: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AV[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AW: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AW[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AX: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AX[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AY: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AY[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AZ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AZ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BA: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BA[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BB: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BB[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BC: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BC[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BD: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BD[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BE: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BE[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BF: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BF[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BG: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BG[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BH: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BH[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BI: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BI[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BJ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BJ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BK: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BK[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BL: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BL[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BM: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BM[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BN: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BN[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BO: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BO[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BP: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BP[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BQ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BQ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BR: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BR[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BS: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BS[index]) - { - encoding_found = true; - break; - } - } - break; - default: - if (fmt == insFmt) - { - encoding_found = true; - index = 0; - } - else - { - encoding_found = false; - } - break; - } - - assert(encoding_found); - const unsigned sve_ins_offset = ((unsigned)ins - INS_sve_invalid); - - switch (index) - { - case 0: - assert(sve_ins_offset < ArrLen(insCodes1)); - code = insCodes1[sve_ins_offset]; - break; - case 1: - assert(sve_ins_offset < ArrLen(insCodes2)); - code = insCodes2[sve_ins_offset]; - break; - case 2: - assert(sve_ins_offset < ArrLen(insCodes3)); - code = insCodes3[sve_ins_offset]; - break; - case 3: - assert(sve_ins_offset < ArrLen(insCodes4)); - code = insCodes4[sve_ins_offset]; - break; - case 4: - assert(sve_ins_offset < ArrLen(insCodes5)); - code = insCodes5[sve_ins_offset]; - break; - case 5: - assert(sve_ins_offset < ArrLen(insCodes6)); - code = insCodes6[sve_ins_offset]; - break; - case 6: - assert(sve_ins_offset < ArrLen(insCodes7)); - code = insCodes7[sve_ins_offset]; - break; - case 7: - assert(sve_ins_offset < ArrLen(insCodes8)); - code = insCodes8[sve_ins_offset]; - break; - case 8: - assert(sve_ins_offset < ArrLen(insCodes9)); - code = insCodes9[sve_ins_offset]; - break; - case 9: - assert(sve_ins_offset < ArrLen(insCodes10)); - code = insCodes10[sve_ins_offset]; - break; - case 10: - assert(sve_ins_offset < ArrLen(insCodes11)); - code = insCodes11[sve_ins_offset]; - break; - case 11: - assert(sve_ins_offset < ArrLen(insCodes12)); - code = insCodes12[sve_ins_offset]; - break; - case 12: - assert(sve_ins_offset < ArrLen(insCodes13)); - code = insCodes13[sve_ins_offset]; - break; - } - - assert((code != BAD_CODE)); - - return code; -} - -// true if this 'imm' can be encoded as a input operand to a mov instruction -/*static*/ bool emitter::emitIns_valid_imm_for_mov(INT64 imm, emitAttr size) -{ - // Check for "MOV (wide immediate)". - if (canEncodeHalfwordImm(imm, size)) - return true; - - // Next try the ones-complement form of 'halfword immediate' imm(i16,hw), - // namely "MOV (inverted wide immediate)". - ssize_t notOfImm = NOT_helper(imm, getBitWidth(size)); - if (canEncodeHalfwordImm(notOfImm, size)) - return true; - - // Finally try "MOV (bitmask immediate)" imm(N,r,s) - if (canEncodeBitMaskImm(imm, size)) - return true; - - return false; -} - -// true if this 'imm' can be encoded as a input operand to a vector movi instruction -/*static*/ bool emitter::emitIns_valid_imm_for_movi(INT64 imm, emitAttr elemsize) -{ - if (elemsize == EA_8BYTE) - { - UINT64 uimm = imm; - while (uimm != 0) - { - INT64 loByte = uimm & 0xFF; - if ((loByte == 0) || (loByte == 0xFF)) - { - uimm >>= 8; - } - else - { - return false; - } - } - assert(uimm == 0); - return true; - } - else - { - // First try the standard 'byteShifted immediate' imm(i8,bySh) - if (canEncodeByteShiftedImm(imm, elemsize, true)) - return true; - - // Next try the ones-complement form of the 'immediate' imm(i8,bySh) - ssize_t notOfImm = NOT_helper(imm, getBitWidth(elemsize)); - if (canEncodeByteShiftedImm(notOfImm, elemsize, true)) - return true; - } - return false; -} - -// true if this 'imm' can be encoded as a input operand to a fmov instruction -/*static*/ bool emitter::emitIns_valid_imm_for_fmov(double immDbl) -{ - if (canEncodeFloatImm8(immDbl)) - return true; - - return false; -} - -// true if this 'imm' can be encoded as a input operand to an add instruction -/*static*/ bool emitter::emitIns_valid_imm_for_add(INT64 imm, emitAttr size) -{ - if (unsigned_abs(imm) <= 0x0fff) - return true; - else if (canEncodeWithShiftImmBy12(imm)) // Try the shifted by 12 encoding - return true; - - return false; -} - -// true if this 'imm' can be encoded as a input operand to an non-add/sub alu instruction -/*static*/ bool emitter::emitIns_valid_imm_for_cmp(INT64 imm, emitAttr size) -{ - return emitIns_valid_imm_for_add(imm, size); -} - -// true if this 'imm' can be encoded as a input operand to an non-add/sub alu instruction -/*static*/ bool emitter::emitIns_valid_imm_for_alu(INT64 imm, emitAttr size) -{ - if (canEncodeBitMaskImm(imm, size)) - return true; - - return false; -} - -// true if this 'imm' can be encoded as the offset in an unscaled ldr/str instruction -/*static*/ bool emitter::emitIns_valid_imm_for_unscaled_ldst_offset(INT64 imm) -{ - return (imm >= -256) && (imm <= 255); -} - -// true if this 'imm' can be encoded as the offset in a ldr/str instruction -/*static*/ bool emitter::emitIns_valid_imm_for_ldst_offset(INT64 imm, emitAttr attr) -{ - if (imm == 0) - return true; // Encodable using IF_LS_2A - - if ((imm >= -256) && (imm <= 255)) - return true; // Encodable using IF_LS_2C (or possibly IF_LS_2B) - - if (imm < 0) - return false; // not encodable - - emitAttr size = EA_SIZE(attr); - unsigned scale = NaturalScale_helper(size); - ssize_t mask = size - 1; // the mask of low bits that must be zero to encode the immediate - - if (((imm & mask) == 0) && ((imm >> scale) < 0x1000)) - return true; // Encodable using IF_LS_2B - - return false; // not encodable -} - -// true if this 'imm' can be encoded as a input operand to a ccmp instruction -/*static*/ bool emitter::emitIns_valid_imm_for_ccmp(INT64 imm) -{ - return ((imm & 0x01f) == imm); -} - -// true if 'imm' can be encoded as an offset in a ldp/stp instruction -/*static*/ bool emitter::canEncodeLoadOrStorePairOffset(INT64 imm, emitAttr attr) -{ - assert((attr == EA_4BYTE) || (attr == EA_8BYTE) || (attr == EA_16BYTE)); - const int size = EA_SIZE_IN_BYTES(attr); - return (imm % size == 0) && (imm >= -64 * size) && (imm < 64 * size); -} - -/************************************************************************ - * - * A helper method to return the natural scale for an EA 'size' - */ - -/*static*/ unsigned emitter::NaturalScale_helper(emitAttr size) -{ - assert(size == EA_1BYTE || size == EA_2BYTE || size == EA_4BYTE || size == EA_8BYTE || size == EA_16BYTE); - return BitOperations::Log2((unsigned)size); -} - -/************************************************************************ - * - * A helper method to perform a Rotate-Right shift operation - * the source is 'value' and it is rotated right by 'sh' bits - * 'value' is considered to be a fixed size 'width' set of bits. - * - * Example - * value is '00001111', sh is 2 and width is 8 - * result is '11000011' - */ - -/*static*/ UINT64 emitter::ROR_helper(UINT64 value, unsigned sh, unsigned width) -{ - assert(width <= 64); - // Check that 'value' fits in 'width' bits - assert((width == 64) || (value < (1ULL << width))); - // We don't support shifts >= width - assert(sh < width); - - UINT64 result; - - unsigned rsh = sh; - unsigned lsh = width - rsh; - - result = (value >> rsh); - result |= (value << lsh); - - if (width < 64) - { - // mask off any extra bits that we got from the left shift - result &= ((1ULL << width) - 1); - } - return result; -} -/************************************************************************ - * - * A helper method to perform a 'NOT' bitwise complement operation. - * 'value' is considered to be a fixed size 'width' set of bits. - * - * Example - * value is '01001011', and width is 8 - * result is '10110100' - */ - -/*static*/ UINT64 emitter::NOT_helper(UINT64 value, unsigned width) -{ - assert(width <= 64); - - UINT64 result = ~value; - - if (width < 64) - { - // Check that 'value' fits in 'width' bits. Don't consider "sign" bits above width. - UINT64 maxVal = 1ULL << width; - UINT64 lowBitsMask = maxVal - 1; - UINT64 signBitsMask = ~lowBitsMask | (1ULL << (width - 1)); // The high bits must be set, and the top bit - // (sign bit) must be set. - assert((value < maxVal) || ((value & signBitsMask) == signBitsMask)); - - // mask off any extra bits that we got from the complement operation - result &= lowBitsMask; - } - - return result; -} - -/************************************************************************ - * - * A helper method to perform a bit Replicate operation - * the source is 'value' with a fixed size 'width' set of bits. - * value is replicated to fill out 8/16/32/64 bits as determined by 'size'. - * - * Example - * value is '11000011' (0xE3), width is 8 and size is EA_8BYTE - * result is '11000011 11000011 11000011 11000011 11000011 11000011 11000011 11000011' - * 0xE3E3E3E3E3E3E3E3 - */ - -/*static*/ UINT64 emitter::Replicate_helper(UINT64 value, unsigned width, emitAttr size) -{ - unsigned immWidth = getBitWidth(size); - assert(width <= immWidth); - - UINT64 result = value; - unsigned filledBits = width; - - while (filledBits < immWidth) - { - value <<= width; - result |= value; - filledBits += width; - } - return result; -} - -/************************************************************************ - * - * Convert an imm(N,r,s) into a 64-bit immediate - * inputs 'bmImm' a bitMaskImm struct - * 'size' specifies the size of the result (8/16/32/64 bits) - */ - -/*static*/ INT64 emitter::emitDecodeBitMaskImm(const emitter::bitMaskImm bmImm, emitAttr size) -{ - unsigned N = bmImm.immN; // read the N,R and S values from the 'bitMaskImm' encoding - unsigned R = bmImm.immR; - unsigned S = bmImm.immS; - - unsigned elemWidth = 64; // used when N == 1 - - if (N == 0) // find the smaller elemWidth when N == 0 - { - // Scan S for the highest bit not set - elemWidth = 32; - for (unsigned bitNum = 5; bitNum > 0; bitNum--) - { - unsigned oneBit = elemWidth; - if ((S & oneBit) == 0) - break; - elemWidth /= 2; - } - } - else - { - assert(size == EA_8BYTE); - } - - unsigned maskSR = elemWidth - 1; - - S &= maskSR; - R &= maskSR; - - // encoding for S is one less than the number of consecutive one bits - S++; // Number of consecutive ones to generate in 'welem' - - // At this point: - // - // 'elemWidth' is the number of bits that we will use for the ROR and Replicate operations - // 'S' is the number of consecutive 1 bits for the immediate - // 'R' is the number of bits that we will Rotate Right the immediate - // 'size' selects the final size of the immediate that we return (64 or 32 bits) - - assert(S < elemWidth); // 'elemWidth' consecutive one's is a reserved encoding - - UINT64 welem; - UINT64 wmask; - - welem = (1ULL << S) - 1; - - wmask = ROR_helper(welem, R, elemWidth); - wmask = Replicate_helper(wmask, elemWidth, size); - - return wmask; -} - -/***************************************************************************** - * - * Check if an immediate can use the left shifted by 12 bits encoding - */ - -/*static*/ bool emitter::canEncodeWithShiftImmBy12(INT64 imm) -{ - if (imm < 0) - { - imm = -imm; // convert to unsigned - } - - if (imm < 0) - { - return false; // Must be MIN_INT64 - } - - if ((imm & 0xfff) != 0) // Now the low 12 bits all have to be zero - { - return false; - } - - imm >>= 12; // shift right by 12 bits - - return (imm <= 0x0fff); // Does it fit in 12 bits -} - -/***************************************************************************** - * - * Normalize the 'imm' so that the upper bits, as defined by 'size' are zero - */ - -/*static*/ INT64 emitter::normalizeImm64(INT64 imm, emitAttr size) -{ - unsigned immWidth = getBitWidth(size); - INT64 result = imm; - - if (immWidth < 64) - { - // Check that 'imm' fits in 'immWidth' bits. Don't consider "sign" bits above width. - INT64 maxVal = 1LL << immWidth; - INT64 lowBitsMask = maxVal - 1; - INT64 hiBitsMask = ~lowBitsMask; - INT64 signBitsMask = - hiBitsMask | (1LL << (immWidth - 1)); // The high bits must be set, and the top bit (sign bit) must be set. - assert((imm < maxVal) || ((imm & signBitsMask) == signBitsMask)); - - // mask off the hiBits - result &= lowBitsMask; - } - return result; -} - -/***************************************************************************** - * - * Normalize the 'imm' so that the upper bits, as defined by 'size' are zero - */ - -/*static*/ INT32 emitter::normalizeImm32(INT32 imm, emitAttr size) -{ - unsigned immWidth = getBitWidth(size); - INT32 result = imm; - - if (immWidth < 32) - { - // Check that 'imm' fits in 'immWidth' bits. Don't consider "sign" bits above width. - INT32 maxVal = 1 << immWidth; - INT32 lowBitsMask = maxVal - 1; - INT32 hiBitsMask = ~lowBitsMask; - INT32 signBitsMask = hiBitsMask | (1 << (immWidth - 1)); // The high bits must be set, and the top bit - // (sign bit) must be set. - assert((imm < maxVal) || ((imm & signBitsMask) == signBitsMask)); - - // mask off the hiBits - result &= lowBitsMask; - } - return result; -} - -/************************************************************************ - * - * returns true if 'imm' of 'size bits (8/16/32/64) can be encoded - * using the ARM64 'bitmask immediate' form. - * When a non-null value is passed for 'wbBMI' then this method - * writes back the 'N','S' and 'R' values use to encode this immediate - * - */ - -/*static*/ bool emitter::canEncodeBitMaskImm(INT64 imm, emitAttr size, emitter::bitMaskImm* wbBMI) -{ - unsigned immWidth = getBitWidth(size); - unsigned maxLen; - - switch (size) - { - case EA_1BYTE: - maxLen = 3; - break; - - case EA_2BYTE: - maxLen = 4; - break; - - case EA_4BYTE: - maxLen = 5; - break; - - case EA_8BYTE: - maxLen = 6; - break; - - default: - assert(!"Invalid size"); - maxLen = 0; - break; - } - - imm = normalizeImm64(imm, size); - - // Starting with len=1, elemWidth is 2 bits - // len=2, elemWidth is 4 bits - // len=3, elemWidth is 8 bits - // len=4, elemWidth is 16 bits - // len=5, elemWidth is 32 bits - // len=6, elemWidth is 64 bits - // - for (unsigned len = 1; (len <= maxLen); len++) - { - unsigned elemWidth = 1 << len; - UINT64 elemMask = ((UINT64)-1) >> (64 - elemWidth); - UINT64 tempImm = (UINT64)imm; // A working copy of 'imm' that we can mutate - UINT64 elemVal = tempImm & elemMask; // The low 'elemWidth' bits of 'imm' - - // Check for all 1's or 0's as these can't be encoded - if ((elemVal == 0) || (elemVal == elemMask)) - continue; - - // 'checkedBits' is the count of bits that are known to match 'elemVal' when replicated - unsigned checkedBits = elemWidth; // by definition the first 'elemWidth' bits match - - // Now check to see if each of the next bits match... - // - while (checkedBits < immWidth) - { - tempImm >>= elemWidth; - - UINT64 nextElem = tempImm & elemMask; - if (nextElem != elemVal) - { - // Not matching, exit this loop and checkedBits will not be equal to immWidth - break; - } - - // The 'nextElem' is matching, so increment 'checkedBits' - checkedBits += elemWidth; - } - - // Did the full immediate contain bits that can be formed by repeating 'elemVal'? - if (checkedBits == immWidth) - { - // We are not quite done, since the only values that we can encode as a - // 'bitmask immediate' are those that can be formed by starting with a - // bit string of 0*1* that is rotated by some number of bits. - // - // We check to see if 'elemVal' can be formed using these restrictions. - // - // Observation: - // Rotating by one bit any value that passes these restrictions - // can be xor-ed with the original value and will result it a string - // of bits that have exactly two 1 bits: 'elemRorXor' - // Further the distance between the two one bits tells us the value - // of S and the location of the 1 bits tells us the value of R - // - // Some examples: (immWidth is 8) - // - // S=4,R=0 S=5,R=3 S=3,R=6 - // elemVal: 00001111 11100011 00011100 - // elemRor: 10000111 11110001 00001110 - // elemRorXor: 10001000 00010010 00010010 - // compute S 45678--- ---5678- ---3210- - // compute R 01234567 ---34567 ------67 - - UINT64 elemRor = ROR_helper(elemVal, 1, elemWidth); // Rotate 'elemVal' Right by one bit - UINT64 elemRorXor = elemVal ^ elemRor; // Xor elemVal and elemRor - - // If we only have a two-bit change in elemROR then we can form a mask for this value - unsigned bitCount = 0; - UINT64 oneBit = 0x1; - unsigned R = elemWidth; // R is shift count for ROR (rotate right shift) - unsigned S = 0; // S is number of consecutive one bits - int incr = -1; - - // Loop over the 'elemWidth' bits in 'elemRorXor' - // - for (unsigned bitNum = 0; bitNum < elemWidth; bitNum++) - { - if (incr == -1) - { - R--; // We decrement R by one whenever incr is -1 - } - if (bitCount == 1) - { - S += incr; // We incr/decr S, after we find the first one bit in 'elemRorXor' - } - - // Is this bit position a 1 bit in 'elemRorXor'? - // - if (oneBit & elemRorXor) - { - bitCount++; - // Is this the first 1 bit that we found in 'elemRorXor'? - if (bitCount == 1) - { - // Does this 1 bit represent a transition to zero bits? - bool toZeros = ((oneBit & elemVal) != 0); - if (toZeros) - { - // S :: Count down from elemWidth - S = elemWidth; - incr = -1; - } - else // this 1 bit represent a transition to one bits. - { - // S :: Count up from zero - S = 0; - incr = +1; - } - } - else // bitCount > 1 - { - // We found the second (or third...) 1 bit in 'elemRorXor' - incr = 0; // stop decrementing 'R' - - if (bitCount > 2) - { - // More than 2 transitions from 0/1 in 'elemVal' - // This means that 'elemVal' can't be encoded - // using a 'bitmask immediate'. - // - // Furthermore, it will continue to fail - // with any larger 'len' that we try. - // so just return false. - // - return false; - } - } - } - - // shift oneBit left by one bit to test the next position - oneBit <<= 1; - } - - // We expect that bitCount will always be two at this point - // but just in case return false for any bad cases. - // - assert(bitCount == 2); - if (bitCount != 2) - return false; - - // Perform some sanity checks on the values of 'S' and 'R' - assert(S > 0); - assert(S < elemWidth); - assert(R < elemWidth); - - // Does the caller want us to return the N,R,S encoding values? - // - if (wbBMI != nullptr) - { - - // The encoding used for S is one less than the - // number of consecutive one bits - S--; - - if (len == 6) - { - wbBMI->immN = 1; - } - else - { - wbBMI->immN = 0; - // The encoding used for 'S' here is a bit peculiar. - // - // The upper bits need to be complemented, followed by a zero bit - // then the value of 'S-1' - // - unsigned upperBitsOfS = 64 - (1 << (len + 1)); - S |= upperBitsOfS; - } - wbBMI->immR = R; - wbBMI->immS = S; - - // Verify that what we are returning is correct. - assert(imm == emitDecodeBitMaskImm(*wbBMI, size)); - } - // Tell the caller that we can successfully encode this immediate - // using a 'bitmask immediate'. - // - return true; - } - } - return false; -} - -/************************************************************************ - * - * Convert a 64-bit immediate into its 'bitmask immediate' representation imm(N,r,s) - */ - -/*static*/ emitter::bitMaskImm emitter::emitEncodeBitMaskImm(INT64 imm, emitAttr size) -{ - emitter::bitMaskImm result; - result.immNRS = 0; - - bool canEncode = canEncodeBitMaskImm(imm, size, &result); - assert(canEncode); - - return result; -} - -/************************************************************************ - * - * Convert an imm(i16,hw) into a 32/64-bit immediate - * inputs 'hwImm' a halfwordImm struct - * 'size' specifies the size of the result (64 or 32 bits) - */ - -/*static*/ INT64 emitter::emitDecodeHalfwordImm(const emitter::halfwordImm hwImm, emitAttr size) -{ - assert(isValidGeneralDatasize(size)); // Only EA_4BYTE or EA_8BYTE forms - - unsigned hw = hwImm.immHW; - INT64 val = (INT64)hwImm.immVal; - - assert((hw <= 1) || (size == EA_8BYTE)); - - INT64 result = val << (16 * hw); - return result; -} - -/************************************************************************ - * - * returns true if 'imm' of 'size' bits (32/64) can be encoded - * using the ARM64 'halfword immediate' form. - * When a non-null value is passed for 'wbHWI' then this method - * writes back the 'immHW' and 'immVal' values use to encode this immediate - * - */ - -/*static*/ bool emitter::canEncodeHalfwordImm(INT64 imm, emitAttr size, emitter::halfwordImm* wbHWI) -{ - assert(isValidGeneralDatasize(size)); // Only EA_4BYTE or EA_8BYTE forms - - unsigned immWidth = (size == EA_8BYTE) ? 64 : 32; - unsigned maxHW = (size == EA_8BYTE) ? 4 : 2; - - // setup immMask to a (EA_4BYTE) 0x00000000_FFFFFFFF or (EA_8BYTE) 0xFFFFFFFF_FFFFFFFF - const UINT64 immMask = ((UINT64)-1) >> (64 - immWidth); - const INT64 mask16 = (INT64)0xFFFF; - - imm = normalizeImm64(imm, size); - - // Try each of the valid hw shift sizes - for (unsigned hw = 0; (hw < maxHW); hw++) - { - INT64 curMask = mask16 << (hw * 16); // Represents the mask of the bits in the current halfword - INT64 checkBits = immMask & ~curMask; - - // Excluding the current halfword (using ~curMask) - // does the immediate have zero bits in every other bit that we care about? - // note we care about all 64-bits for EA_8BYTE - // and we care about the lowest 32 bits for EA_4BYTE - // - if ((imm & checkBits) == 0) - { - // Does the caller want us to return the imm(i16,hw) encoding values? - // - if (wbHWI != nullptr) - { - INT64 val = ((imm & curMask) >> (hw * 16)) & mask16; - wbHWI->immHW = hw; - wbHWI->immVal = val; - - // Verify that what we are returning is correct. - assert(imm == emitDecodeHalfwordImm(*wbHWI, size)); - } - // Tell the caller that we can successfully encode this immediate - // using a 'halfword immediate'. - // - return true; - } - } - return false; -} - -/************************************************************************ - * - * Convert a 64-bit immediate into its 'halfword immediate' representation imm(i16,hw) - */ - -/*static*/ emitter::halfwordImm emitter::emitEncodeHalfwordImm(INT64 imm, emitAttr size) -{ - emitter::halfwordImm result; - result.immHWVal = 0; - - bool canEncode = canEncodeHalfwordImm(imm, size, &result); - assert(canEncode); - - return result; -} - -/************************************************************************ - * - * Convert an imm(i8,sh) into a 16/32-bit immediate - * inputs 'bsImm' a byteShiftedImm struct - * 'size' specifies the size of the result (16 or 32 bits) - */ - -/*static*/ UINT32 emitter::emitDecodeByteShiftedImm(const emitter::byteShiftedImm bsImm, emitAttr size) -{ - bool onesShift = (bsImm.immOnes == 1); - unsigned bySh = bsImm.immBY; // Num Bytes to shift 0,1,2,3 - UINT32 result = (UINT32)bsImm.immVal; // 8-bit immediate - - if (bySh > 0) - { - assert((size == EA_2BYTE) || (size == EA_4BYTE)); // Only EA_2BYTE or EA_4BYTE forms - if (size == EA_2BYTE) - { - assert(bySh < 2); - } - else - { - assert(bySh < 4); - } - - result <<= (8 * bySh); - - if (onesShift) - { - result |= ((1 << (8 * bySh)) - 1); - } - } - return result; -} - -/************************************************************************ - * - * returns true if 'imm' of 'size' bits (16/32) can be encoded - * using the ARM64 'byteShifted immediate' form. - * When a non-null value is passed for 'wbBSI' then this method - * writes back the 'immBY' and 'immVal' values use to encode this immediate - * - */ - -/*static*/ bool emitter::canEncodeByteShiftedImm(INT64 imm, - emitAttr size, - bool allow_MSL, - emitter::byteShiftedImm* wbBSI) -{ - bool canEncode = false; - bool onesShift = false; // true if we use the shifting ones variant - unsigned bySh = 0; // number of bytes to shift: 0, 1, 2, 3 - unsigned imm8 = 0; // immediate to use in the encoding - - imm = normalizeImm64(imm, size); - - if (size == EA_1BYTE) - { - imm8 = (unsigned)imm; - assert(imm8 < 0x100); - canEncode = true; - } - else if (size == EA_8BYTE) - { - imm8 = (unsigned)imm; - assert(imm8 < 0x100); - canEncode = true; - } - else - { - assert((size == EA_2BYTE) || (size == EA_4BYTE)); // Only EA_2BYTE or EA_4BYTE forms - - unsigned immWidth = (size == EA_4BYTE) ? 32 : 16; - unsigned maxBY = (size == EA_4BYTE) ? 4 : 2; - - // setup immMask to a (EA_2BYTE) 0x0000FFFF or (EA_4BYTE) 0xFFFFFFFF - const UINT32 immMask = ((UINT32)-1) >> (32 - immWidth); - const INT32 mask8 = (INT32)0xFF; - - // Try each of the valid by shift sizes - for (bySh = 0; (bySh < maxBY); bySh++) - { - INT32 curMask = mask8 << (bySh * 8); // Represents the mask of the bits in the current byteShifted - INT32 checkBits = immMask & ~curMask; - INT32 immCheck = (imm & checkBits); - - // Excluding the current byte (using ~curMask) - // does the immediate have zero bits in every other bit that we care about? - // or can be use the shifted one variant? - // note we care about all 32-bits for EA_4BYTE - // and we care about the lowest 16 bits for EA_2BYTE - // - if (immCheck == 0) - { - canEncode = true; - } - - // MSL is only supported for 32-bit. - if (allow_MSL && (size == EA_4BYTE)) - { - if ((bySh == 1) && (immCheck == 0xFF)) - { - canEncode = true; - onesShift = true; - } - else if ((bySh == 2) && (immCheck == 0xFFFF)) - { - canEncode = true; - onesShift = true; - } - } - if (canEncode) - { - imm8 = (unsigned)(((imm & curMask) >> (bySh * 8)) & mask8); - break; - } - } - } - - if (canEncode) - { - // Does the caller want us to return the imm(i8,bySh) encoding values? - // - if (wbBSI != nullptr) - { - wbBSI->immOnes = onesShift; - wbBSI->immBY = bySh; - wbBSI->immVal = imm8; - - // Verify that what we are returning is correct. - assert(imm == emitDecodeByteShiftedImm(*wbBSI, size)); - } - // Tell the caller that we can successfully encode this immediate - // using a 'byteShifted immediate'. - // - return true; - } - return false; -} - -/************************************************************************ - * - * Convert a 32-bit immediate into its 'byteShifted immediate' representation imm(i8,by) - */ - -/*static*/ emitter::byteShiftedImm emitter::emitEncodeByteShiftedImm(INT64 imm, emitAttr size, bool allow_MSL) -{ - emitter::byteShiftedImm result; - result.immBSVal = 0; - - bool canEncode = canEncodeByteShiftedImm(imm, size, allow_MSL, &result); - assert(canEncode); - - return result; -} - -/************************************************************************ - * - * Convert a 'float 8-bit immediate' into a double. - * inputs 'fpImm' a floatImm8 struct - */ - -/*static*/ double emitter::emitDecodeFloatImm8(const emitter::floatImm8 fpImm) -{ - unsigned sign = fpImm.immSign; - unsigned exp = fpImm.immExp ^ 0x4; - unsigned mant = fpImm.immMant + 16; - unsigned scale = 16 * 8; - - while (exp > 0) - { - scale /= 2; - exp--; - } - - double result = ((double)mant) / ((double)scale); - if (sign == 1) - { - result = -result; - } - - return result; -} - -/************************************************************************ - * - * returns true if the 'immDbl' can be encoded using the 'float 8-bit immediate' form. - * also returns the encoding if wbFPI is non-null - * - */ - -/*static*/ bool emitter::canEncodeFloatImm8(double immDbl, emitter::floatImm8* wbFPI) -{ - bool canEncode = false; - double val = immDbl; - - int sign = 0; - if (val < 0.0) - { - val = -val; - sign = 1; - } - - int exp = 0; - while ((val < 1.0) && (exp >= -4)) - { - val *= 2.0; - exp--; - } - while ((val >= 2.0) && (exp <= 5)) - { - val *= 0.5; - exp++; - } - exp += 3; - val *= 16.0; - int ival = (int)val; - - if ((exp >= 0) && (exp <= 7)) - { - if (val == (double)ival) - { - canEncode = true; - - if (wbFPI != nullptr) - { - ival -= 16; - assert((ival >= 0) && (ival <= 15)); - - wbFPI->immSign = sign; - wbFPI->immExp = exp ^ 0x4; - wbFPI->immMant = ival; - unsigned imm8 = wbFPI->immFPIVal; - assert((imm8 >= 0) && (imm8 <= 0xff)); - } - } - } - - return canEncode; -} - -/************************************************************************ - * - * Convert a double into its 'float 8-bit immediate' representation - */ - -/*static*/ emitter::floatImm8 emitter::emitEncodeFloatImm8(double immDbl) -{ - emitter::floatImm8 result; - result.immFPIVal = 0; - - bool canEncode = canEncodeFloatImm8(immDbl, &result); - assert(canEncode); - - return result; -} - -/************************************************************************ - * - * Convert a rotation value that is 90 or 270 into a smaller encoding that matches one-to-one with the 'rot' field. - */ - -/*static*/ ssize_t emitter::emitEncodeRotationImm90_or_270(ssize_t imm) -{ - switch (imm) - { - case 90: - return 0; - - case 270: - return 1; - - default: - break; - } - - assert(!"Invalid rotation value"); - return 0; -} - -/************************************************************************ - * - * Convert an encoded rotation value to 90 or 270. - */ - -/*static*/ ssize_t emitter::emitDecodeRotationImm90_or_270(ssize_t imm) -{ - assert(emitIsValidEncodedRotationImm0_to_270(imm)); - switch (imm) - { - case 0: - return 90; - - case 1: - return 270; - - default: - break; - } - - return 0; -} - -/************************************************************************ - * - * Check if the immediate value is a valid encoded rotation value for 90 or 270. - */ - -/*static*/ bool emitter::emitIsValidEncodedRotationImm90_or_270(ssize_t imm) -{ - return (imm == 0) || (imm == 1); -} - -/************************************************************************ - * - * Convert a rotation value that is 0, 90, 180 or 270 into a smaller encoding that matches one-to-one with the 'rot' - * field. - */ - -/*static*/ ssize_t emitter::emitEncodeRotationImm0_to_270(ssize_t imm) -{ - switch (imm) - { - case 0: - return 0; - - case 90: - return 1; - - case 180: - return 2; - - case 270: - return 3; - - default: - break; - } - - assert(!"Invalid rotation value"); - return 0; -} - -/************************************************************************ - * - * Convert an encoded rotation value to 0, 90, 180 or 270. - */ - -/*static*/ ssize_t emitter::emitDecodeRotationImm0_to_270(ssize_t imm) -{ - assert(emitIsValidEncodedRotationImm0_to_270(imm)); - switch (imm) - { - case 0: - return 0; - - case 1: - return 90; - - case 2: - return 180; - - case 3: - return 270; - - default: - break; - } - - return 0; -} - -/************************************************************************ - * - * Check if the immediate value is a valid encoded rotation value for 0, 90, 180 or 270. - */ - -/*static*/ bool emitter::emitIsValidEncodedRotationImm0_to_270(ssize_t imm) -{ - return (imm >= 0) && (imm <= 3); -} - -/************************************************************************ - * - * Convert a small immediate float value to an encoded version that matches one-to-one with the instructions. - * The instruction determines the value. - */ - -/*static*/ ssize_t emitter::emitEncodeSmallFloatImm(double immDbl, instruction ins) -{ -#ifdef DEBUG - switch (ins) - { - case INS_sve_fadd: - case INS_sve_fsub: - case INS_sve_fsubr: - assert((immDbl == 0.5) || (immDbl == 1.0)); - break; - - case INS_sve_fmax: - case INS_sve_fmaxnm: - case INS_sve_fmin: - case INS_sve_fminnm: - assert((immDbl == 0) || (immDbl == 1.0)); - break; - - case INS_sve_fmul: - assert((immDbl == 0.5) || (immDbl == 2.0)); - break; - - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG - if (immDbl < 1.0) - { - return 0; - } - return 1; -} - -/************************************************************************ - * - * Convert an encoded small float immediate value. The instruction determines the value. - */ - -/*static*/ double emitter::emitDecodeSmallFloatImm(ssize_t imm, instruction ins) -{ - assert(emitIsValidEncodedSmallFloatImm(imm)); - switch (ins) - { - case INS_sve_fadd: - case INS_sve_fsub: - case INS_sve_fsubr: - if (imm == 0) - { - return 0.5; - } - else - { - return 1.0; - } - - case INS_sve_fmax: - case INS_sve_fmaxnm: - case INS_sve_fmin: - case INS_sve_fminnm: - if (imm == 0) - { - return 0.0; - } - else - { - return 1.0; - } - break; - - case INS_sve_fmul: - if (imm == 0) - { - return 0.5; - } - else - { - return 2.0; - } - break; - - default: - break; - } - - assert(!"Invalid instruction"); - return 0.0; -} - -/************************************************************************ - * - * Check if the immediate value is a valid encoded small float. - */ - -/*static*/ bool emitter::emitIsValidEncodedSmallFloatImm(size_t imm) -{ - return (imm == 0) || (imm == 1); -} - -/***************************************************************************** - * - * For the given 'ins' returns the reverse instruction - * if one exists, otherwise returns INS_INVALID - */ - -/*static*/ instruction emitter::insReverse(instruction ins) -{ - switch (ins) - { - case INS_add: - return INS_sub; - case INS_adds: - return INS_subs; - - case INS_sub: - return INS_add; - case INS_subs: - return INS_adds; - - case INS_cmp: - return INS_cmn; - case INS_cmn: - return INS_cmp; - - case INS_ccmp: - return INS_ccmn; - case INS_ccmn: - return INS_ccmp; - - default: - return INS_invalid; - } -} - -/***************************************************************************** - * - * For the given 'datasize' and 'elemsize', make the proper arrangement option - * returns the insOpts that specifies the vector register arrangement - * if one does not exist returns INS_OPTS_NONE - */ - -/*static*/ insOpts emitter::optMakeArrangement(emitAttr datasize, emitAttr elemsize) -{ - insOpts result = INS_OPTS_NONE; - - if (datasize == EA_8BYTE) - { - switch (elemsize) - { - case EA_1BYTE: - result = INS_OPTS_8B; - break; - case EA_2BYTE: - result = INS_OPTS_4H; - break; - case EA_4BYTE: - result = INS_OPTS_2S; - break; - case EA_8BYTE: - result = INS_OPTS_1D; - break; - default: - unreached(); - break; - } - } - else if (datasize == EA_16BYTE) - { - switch (elemsize) - { - case EA_1BYTE: - result = INS_OPTS_16B; - break; - case EA_2BYTE: - result = INS_OPTS_8H; - break; - case EA_4BYTE: - result = INS_OPTS_4S; - break; - case EA_8BYTE: - result = INS_OPTS_2D; - break; - default: - unreached(); - break; - } - } - return result; -} - -/***************************************************************************** - * - * For the given 'datasize' and arrangement 'opts' - * returns true is the pair specifies a valid arrangement - */ -/*static*/ bool emitter::isValidArrangement(emitAttr datasize, insOpts opt) -{ - if (datasize == EA_8BYTE) - { - if ((opt == INS_OPTS_8B) || (opt == INS_OPTS_4H) || (opt == INS_OPTS_2S) || (opt == INS_OPTS_1D)) - { - return true; - } - } - else if (datasize == EA_16BYTE) - { - if ((opt == INS_OPTS_16B) || (opt == INS_OPTS_8H) || (opt == INS_OPTS_4S) || (opt == INS_OPTS_2D)) - { - return true; - } - } - return false; -} - -//------------------------------------------------------------------------ -// insGetRegisterListSize: Returns a size of the register list a given instruction operates on. -// -// Arguments: -// ins - An instruction which uses a register list -// (e.g. ld1 (2 registers), ld1r, st1, tbl, tbx). -// -// Return value: -// A number of consecutive SIMD and floating-point registers the instruction loads to/store from. -// -/*static*/ unsigned emitter::insGetRegisterListSize(instruction ins) -{ - unsigned registerListSize = 0; - - switch (ins) - { - case INS_ld1: - case INS_ld1r: - case INS_st1: - case INS_tbl: - case INS_tbx: - registerListSize = 1; - break; - - case INS_ld1_2regs: - case INS_ld2: - case INS_ld2r: - case INS_st1_2regs: - case INS_st2: - case INS_tbl_2regs: - case INS_tbx_2regs: - registerListSize = 2; - break; - - case INS_ld1_3regs: - case INS_ld3: - case INS_ld3r: - case INS_st1_3regs: - case INS_st3: - case INS_tbl_3regs: - case INS_tbx_3regs: - registerListSize = 3; - break; - - case INS_ld1_4regs: - case INS_ld4: - case INS_ld4r: - case INS_st1_4regs: - case INS_st4: - case INS_tbl_4regs: - case INS_tbx_4regs: - registerListSize = 4; - break; - - default: - assert(!"Unexpected instruction"); - break; - } - - return registerListSize; -} - -/***************************************************************************** - * - * Expands an option that has different size operands (INS_OPTS_*_TO_*) into - * a pair of scalable options where the first describes the size of the - * destination operand and the second describes the size of the source operand. - */ - -/*static*/ void emitter::optExpandConversionPair(insOpts opt, insOpts& dst, insOpts& src) -{ - dst = INS_OPTS_NONE; - src = INS_OPTS_NONE; - - switch (opt) - { - case INS_OPTS_H_TO_S: - dst = INS_OPTS_SCALABLE_S; - src = INS_OPTS_SCALABLE_H; - break; - case INS_OPTS_S_TO_H: - dst = INS_OPTS_SCALABLE_H; - src = INS_OPTS_SCALABLE_S; - break; - case INS_OPTS_S_TO_D: - dst = INS_OPTS_SCALABLE_D; - src = INS_OPTS_SCALABLE_S; - break; - case INS_OPTS_D_TO_S: - dst = INS_OPTS_SCALABLE_S; - src = INS_OPTS_SCALABLE_D; - break; - case INS_OPTS_H_TO_D: - dst = INS_OPTS_SCALABLE_D; - src = INS_OPTS_SCALABLE_H; - break; - case INS_OPTS_D_TO_H: - dst = INS_OPTS_SCALABLE_H; - src = INS_OPTS_SCALABLE_D; - break; - case INS_OPTS_SCALABLE_H: - dst = INS_OPTS_SCALABLE_H; - src = INS_OPTS_SCALABLE_H; - break; - case INS_OPTS_SCALABLE_S: - dst = INS_OPTS_SCALABLE_S; - src = INS_OPTS_SCALABLE_S; - break; - case INS_OPTS_SCALABLE_D: - dst = INS_OPTS_SCALABLE_D; - src = INS_OPTS_SCALABLE_D; - break; - default: - noway_assert(!"unreachable"); - break; - } - - assert(dst != INS_OPTS_NONE && src != INS_OPTS_NONE); - return; -} - -// For the given 'arrangement' returns the 'datasize' specified by the vector register arrangement -// asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed -// -/*static*/ emitAttr emitter::optGetDatasize(insOpts arrangement) -{ - if ((arrangement == INS_OPTS_8B) || (arrangement == INS_OPTS_4H) || (arrangement == INS_OPTS_2S) || - (arrangement == INS_OPTS_1D)) - { - return EA_8BYTE; - } - else if ((arrangement == INS_OPTS_16B) || (arrangement == INS_OPTS_8H) || (arrangement == INS_OPTS_4S) || - (arrangement == INS_OPTS_2D)) - { - return EA_16BYTE; - } - else - { - assert(!" invalid 'arrangement' value"); - return EA_UNKNOWN; - } -} - -// For the given 'arrangement' returns the 'elemsize' specified by the vector register arrangement -// asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed -// -/*static*/ emitAttr emitter::optGetElemsize(insOpts arrangement) -{ - if ((arrangement == INS_OPTS_8B) || (arrangement == INS_OPTS_16B)) - { - return EA_1BYTE; - } - else if ((arrangement == INS_OPTS_4H) || (arrangement == INS_OPTS_8H)) - { - return EA_2BYTE; - } - else if ((arrangement == INS_OPTS_2S) || (arrangement == INS_OPTS_4S)) - { - return EA_4BYTE; - } - else if ((arrangement == INS_OPTS_1D) || (arrangement == INS_OPTS_2D)) - { - return EA_8BYTE; - } - else - { - assert(!" invalid 'arrangement' value"); - return EA_UNKNOWN; - } -} - -// For the given 'elemsize' returns the 'arrangement' when used in a SVE vector register arrangement. -// Asserts and returns INS_OPTS_NONE if an invalid 'elemsize' is passed -// -/*static*/ insOpts emitter::optGetSveInsOpt(emitAttr elemsize) -{ - switch (elemsize) - { - case EA_1BYTE: - return INS_OPTS_SCALABLE_B; - - case EA_2BYTE: - return INS_OPTS_SCALABLE_H; - - case EA_4BYTE: - return INS_OPTS_SCALABLE_S; - - case EA_8BYTE: - return INS_OPTS_SCALABLE_D; - - case EA_16BYTE: - return INS_OPTS_SCALABLE_Q; - - default: - assert(!"Invalid emitAttr for sve vector register"); - return INS_OPTS_NONE; - } -} - -// For the given 'arrangement' returns the 'elemsize' specified by the SVE vector register arrangement -// asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed -// -/*static*/ emitAttr emitter::optGetSveElemsize(insOpts arrangement) -{ - switch (arrangement) - { - case INS_OPTS_SCALABLE_B: - return EA_1BYTE; - - case INS_OPTS_SCALABLE_H: - return EA_2BYTE; - - case INS_OPTS_SCALABLE_S: - case INS_OPTS_SCALABLE_S_UXTW: - case INS_OPTS_SCALABLE_S_SXTW: - return EA_4BYTE; - - case INS_OPTS_SCALABLE_D: - case INS_OPTS_SCALABLE_D_UXTW: - case INS_OPTS_SCALABLE_D_SXTW: - return EA_8BYTE; - - case INS_OPTS_SCALABLE_Q: - return EA_16BYTE; - - default: - assert(!"Invalid insOpt for vector register"); - return EA_UNKNOWN; - } -} - -/*static*/ insOpts emitter::optWidenElemsizeArrangement(insOpts arrangement) -{ - if ((arrangement == INS_OPTS_8B) || (arrangement == INS_OPTS_16B)) - { - return INS_OPTS_8H; - } - else if ((arrangement == INS_OPTS_4H) || (arrangement == INS_OPTS_8H)) - { - return INS_OPTS_4S; - } - else if ((arrangement == INS_OPTS_2S) || (arrangement == INS_OPTS_4S)) - { - return INS_OPTS_2D; - } - else - { - assert(!" invalid 'arrangement' value"); - return INS_OPTS_NONE; - } -} - -/*static*/ insOpts emitter::optWidenSveElemsizeArrangement(insOpts arrangement) -{ - switch (arrangement) - { - case INS_OPTS_SCALABLE_B: - return INS_OPTS_SCALABLE_H; - - case INS_OPTS_SCALABLE_H: - return INS_OPTS_SCALABLE_S; - - case INS_OPTS_SCALABLE_S: - return INS_OPTS_SCALABLE_D; - - default: - assert(!" invalid 'arrangement' value"); - return INS_OPTS_NONE; - } -} - -/*static*/ insOpts emitter::optSveToQuadwordElemsizeArrangement(insOpts arrangement) -{ - switch (arrangement) - { - case INS_OPTS_SCALABLE_B: - return INS_OPTS_16B; - - case INS_OPTS_SCALABLE_H: - return INS_OPTS_8H; - - case INS_OPTS_SCALABLE_S: - return INS_OPTS_4S; - - case INS_OPTS_SCALABLE_D: - return INS_OPTS_2D; - - default: - assert(!" invalid 'arrangement' value"); - return INS_OPTS_NONE; - } -} - -/*static*/ emitAttr emitter::widenDatasize(emitAttr datasize) -{ - if (datasize == EA_1BYTE) - { - return EA_2BYTE; - } - else if (datasize == EA_2BYTE) - { - return EA_4BYTE; - } - else if (datasize == EA_4BYTE) - { - return EA_8BYTE; - } - else - { - assert(!" invalid 'datasize' value"); - return EA_UNKNOWN; - } -} - -// For the given 'srcArrangement' returns the "widen" 'dstArrangement' specifying the destination vector register -// arrangement -// asserts and returns INS_OPTS_NONE if an invalid 'srcArrangement' value is passed -// -/*static*/ insOpts emitter::optWidenDstArrangement(insOpts srcArrangement) -{ - insOpts dstArrangement = INS_OPTS_NONE; - - switch (srcArrangement) - { - case INS_OPTS_8B: - dstArrangement = INS_OPTS_4H; - break; - - case INS_OPTS_16B: - dstArrangement = INS_OPTS_8H; - break; - - case INS_OPTS_4H: - dstArrangement = INS_OPTS_2S; - break; - - case INS_OPTS_8H: - dstArrangement = INS_OPTS_4S; - break; - - case INS_OPTS_2S: - dstArrangement = INS_OPTS_1D; - break; - - case INS_OPTS_4S: - dstArrangement = INS_OPTS_2D; - break; - - default: - assert(!" invalid 'srcArrangement' value"); - break; - } - - return dstArrangement; -} - -// For the given 'conversion' returns the 'dstsize' specified by the conversion option -/*static*/ emitAttr emitter::optGetDstsize(insOpts conversion) -{ - switch (conversion) - { - case INS_OPTS_S_TO_8BYTE: - case INS_OPTS_D_TO_8BYTE: - case INS_OPTS_4BYTE_TO_D: - case INS_OPTS_8BYTE_TO_D: - case INS_OPTS_S_TO_D: - case INS_OPTS_H_TO_D: - - return EA_8BYTE; - - case INS_OPTS_S_TO_4BYTE: - case INS_OPTS_D_TO_4BYTE: - case INS_OPTS_4BYTE_TO_S: - case INS_OPTS_8BYTE_TO_S: - case INS_OPTS_D_TO_S: - case INS_OPTS_H_TO_S: - - return EA_4BYTE; - - case INS_OPTS_S_TO_H: - case INS_OPTS_D_TO_H: - - return EA_2BYTE; - - default: - assert(!" invalid 'conversion' value"); - return EA_UNKNOWN; - } -} - -// For the given 'conversion' returns the 'srcsize' specified by the conversion option -/*static*/ emitAttr emitter::optGetSrcsize(insOpts conversion) -{ - switch (conversion) - { - case INS_OPTS_D_TO_8BYTE: - case INS_OPTS_D_TO_4BYTE: - case INS_OPTS_8BYTE_TO_D: - case INS_OPTS_8BYTE_TO_S: - case INS_OPTS_D_TO_S: - case INS_OPTS_D_TO_H: - - return EA_8BYTE; - - case INS_OPTS_S_TO_8BYTE: - case INS_OPTS_S_TO_4BYTE: - case INS_OPTS_4BYTE_TO_S: - case INS_OPTS_4BYTE_TO_D: - case INS_OPTS_S_TO_D: - case INS_OPTS_S_TO_H: - - return EA_4BYTE; - - case INS_OPTS_H_TO_S: - case INS_OPTS_H_TO_D: - - return EA_2BYTE; - - default: - assert(!" invalid 'conversion' value"); - return EA_UNKNOWN; - } -} - -// For the given 'size' and 'index' returns true if it specifies a valid index for a vector register of 'size' -/*static*/ bool emitter::isValidVectorIndex(emitAttr datasize, emitAttr elemsize, ssize_t index) -{ - assert(isValidVectorDatasize(datasize)); - assert(isValidVectorElemsize(elemsize)); - - bool result = false; - if (index >= 0) - { - if (datasize == EA_8BYTE) - { - switch (elemsize) - { - case EA_1BYTE: - result = (index < 8); - break; - case EA_2BYTE: - result = (index < 4); - break; - case EA_4BYTE: - result = (index < 2); - break; - case EA_8BYTE: - result = (index < 1); - break; - default: - unreached(); - break; - } - } - else if (datasize == EA_16BYTE) - { - switch (elemsize) - { - case EA_1BYTE: - result = (index < 16); - break; - case EA_2BYTE: - result = (index < 8); - break; - case EA_4BYTE: - result = (index < 4); - break; - case EA_8BYTE: - result = (index < 2); - break; - default: - unreached(); - break; - } - } - } - return result; -} - -/***************************************************************************** - * - * Add an instruction with no operands. - */ - -void emitter::emitIns(instruction ins) -{ - instrDesc* id = emitNewInstrSmall(EA_8BYTE); - insFormat fmt = emitInsFormat(ins); - - if (ins != INS_BREAKPOINT) - { - assert(fmt == IF_SN_0A); - } - - id->idIns(ins); - id->idInsFmt(fmt); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction with a single immediate value. - */ - -void emitter::emitIns_I(instruction ins, emitAttr attr, ssize_t imm) -{ - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - if (ins == INS_BREAKPOINT) - { - if ((imm & 0x0000ffff) == imm) - { - fmt = IF_SI_0A; - } - else - { - assert(!"Instruction cannot be encoded: IF_SI_0A"); - } - } - else - { - // fallback to emit SVE instructions. - return emitInsSve_I(ins, attr, imm); - } - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSC(attr, imm); - - id->idIns(ins); - id->idInsFmt(fmt); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing a single register. - */ - -void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg, insOpts opt /* = INS_OPTS_NONE */) -{ - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_br: - case INS_ret: - assert(isGeneralRegister(reg)); - fmt = IF_BR_1A; - break; - - case INS_dczva: - assert(isGeneralRegister(reg)); - assert(attr == EA_8BYTE); - fmt = IF_SR_1A; - break; - - case INS_mrs_tpid0: - fmt = IF_SR_1A; - break; - - default: - // fallback to emit SVE instructions. - return emitInsSve_R(ins, attr, reg, opt); - } - - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSmall(attr); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idReg1(reg); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing a register and a constant. - */ - -void emitter::emitIns_R_I(instruction ins, - emitAttr attr, - regNumber reg, - ssize_t imm, - insOpts opt, /* = INS_OPTS_NONE */ - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */ - DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = GTF_EMPTY */)) -{ - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - bool canEncode = false; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - bitMaskImm bmi; - halfwordImm hwi; - byteShiftedImm bsi; - ssize_t notOfImm; - - case INS_tst: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg)); - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, size, &bmi); - if (canEncode) - { - imm = bmi.immNRS; - assert(isValidImmNRS(imm, size)); - fmt = IF_DI_1C; - } - break; - - case INS_movk: - case INS_movn: - case INS_movz: - assert(isValidGeneralDatasize(size)); - assert(insOptsNone(opt)); // No LSL here (you must use emitIns_R_I_I if a shift is needed) - assert(isGeneralRegister(reg)); - assert(isValidUimm<16>(imm)); - - hwi.immHW = 0; - hwi.immVal = imm; - assert(imm == emitDecodeHalfwordImm(hwi, size)); - - imm = hwi.immHWVal; - canEncode = true; - fmt = IF_DI_1B; - break; - - case INS_mov: - assert(isValidGeneralDatasize(size)); - assert(insOptsNone(opt)); // No explicit LSL here - // We will automatically determine the shift based upon the imm - - // First try the standard 'halfword immediate' imm(i16,hw) - hwi.immHWVal = 0; - canEncode = canEncodeHalfwordImm(imm, size, &hwi); - if (canEncode) - { - // uses a movz encoding - assert(isGeneralRegister(reg)); - imm = hwi.immHWVal; - assert(isValidImmHWVal(imm, size)); - fmt = IF_DI_1B; - break; - } - - // Next try the ones-complement form of 'halfword immediate' imm(i16,hw) - notOfImm = NOT_helper(imm, getBitWidth(size)); - canEncode = canEncodeHalfwordImm(notOfImm, size, &hwi); - if (canEncode) - { - assert(isGeneralRegister(reg)); - imm = hwi.immHWVal; - ins = INS_movn; // uses a movn encoding - assert(isValidImmHWVal(imm, size)); - fmt = IF_DI_1B; - break; - } - - // Finally try the 'bitmask immediate' imm(N,r,s) - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, size, &bmi); - if (canEncode) - { - assert(isGeneralRegisterOrSP(reg)); - reg = encodingSPtoZR(reg); - imm = bmi.immNRS; - assert(isValidImmNRS(imm, size)); - fmt = IF_DI_1D; - break; - } - else - { - assert(!"Instruction cannot be encoded: mov imm"); - } - - break; - - case INS_movi: - assert(isValidVectorDatasize(size)); - assert(isVectorRegister(reg)); - if (insOptsNone(opt) && (size == EA_8BYTE)) - { - opt = INS_OPTS_1D; - } - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - - if (elemsize == EA_8BYTE) - { - size_t uimm = imm; - ssize_t imm8 = 0; - unsigned pos = 0; - canEncode = true; - while (uimm != 0) - { - INT64 loByte = uimm & 0xFF; - if (((loByte == 0) || (loByte == 0xFF)) && (pos < 8)) - { - if (loByte == 0xFF) - { - imm8 |= (ssize_t{1} << pos); - } - uimm >>= 8; - pos++; - } - else - { - canEncode = false; - break; - } - } - imm = imm8; - assert(isValidUimm<8>(imm)); - fmt = IF_DV_1B; - break; - } - else - { - // Vector operation - - // No explicit LSL/MSL is used for the immediate - // We will automatically determine the shift based upon the value of imm - - // First try the standard 'byteShifted immediate' imm(i8,bySh) - bsi.immBSVal = 0; - canEncode = canEncodeByteShiftedImm(imm, elemsize, true, &bsi); - if (canEncode) - { - imm = bsi.immBSVal; - assert(isValidImmBSVal(imm, size)); - fmt = IF_DV_1B; - break; - } - - // Next try the ones-complement form of the 'immediate' imm(i8,bySh) - if ((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)) // Only EA_2BYTE or EA_4BYTE forms - { - notOfImm = NOT_helper(imm, getBitWidth(elemsize)); - canEncode = canEncodeByteShiftedImm(notOfImm, elemsize, true, &bsi); - if (canEncode) - { - imm = bsi.immBSVal; - ins = INS_mvni; // uses a mvni encoding - assert(isValidImmBSVal(imm, size)); - fmt = IF_DV_1B; - break; - } - } - } - break; - - case INS_orr: - case INS_bic: - case INS_mvni: - assert(isValidVectorDatasize(size)); - assert(isVectorRegister(reg)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)); // Only EA_2BYTE or EA_4BYTE forms - - // Vector operation - - // No explicit LSL/MSL is used for the immediate - // We will automatically determine the shift based upon the value of imm - - // First try the standard 'byteShifted immediate' imm(i8,bySh) - bsi.immBSVal = 0; - canEncode = canEncodeByteShiftedImm(imm, elemsize, - (ins == INS_mvni), // mvni supports the ones shifting variant (aka MSL) - &bsi); - if (canEncode) - { - imm = bsi.immBSVal; - assert(isValidImmBSVal(imm, size)); - fmt = IF_DV_1B; - break; - } - break; - - case INS_cmp: - case INS_cmn: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg)); - - if (unsigned_abs(imm) <= 0x0fff) - { - if (imm < 0) - { - ins = insReverse(ins); - imm = -imm; - } - assert(isValidUimm<12>(imm)); - canEncode = true; - fmt = IF_DI_1A; - } - else if (canEncodeWithShiftImmBy12(imm)) // Try the shifted by 12 encoding - { - // Encoding will use a 12-bit left shift of the immediate - opt = INS_OPTS_LSL12; - if (imm < 0) - { - ins = insReverse(ins); - imm = -imm; - } - assert((imm & 0xfff) == 0); - imm >>= 12; - assert(isValidUimm<12>(imm)); - canEncode = true; - fmt = IF_DI_1A; - } - else - { - assert(!"Instruction cannot be encoded: IF_DI_1A"); - } - break; - - default: - // fallback to emit SVE instructions. - return emitInsSve_R_I(ins, attr, reg, imm, opt, sopt); - } // end switch (ins) - - assert(canEncode); - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSC(attr, imm); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg); - -#ifdef DEBUG - id->idDebugOnlyInfo()->idMemCookie = targetHandle; - id->idDebugOnlyInfo()->idFlags = gtFlags; -#endif - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing a register and a floating point constant. - */ - -void emitter::emitIns_R_F( - instruction ins, emitAttr attr, regNumber reg, double immDbl, insOpts opt /* = INS_OPTS_NONE */) - -{ - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - ssize_t imm = 0; - bool canEncode = false; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - floatImm8 fpi; - - case INS_fcmp: - case INS_fcmpe: - assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); - assert(isVectorRegister(reg)); - if (immDbl == 0.0) - { - canEncode = true; - fmt = IF_DV_1C; - } - break; - - case INS_fmov: - assert(isVectorRegister(reg)); - fpi.immFPIVal = 0; - canEncode = canEncodeFloatImm8(immDbl, &fpi); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsizeFloat(elemsize)); - assert(opt != INS_OPTS_1D); // Reserved encoding - - if (canEncode) - { - imm = fpi.immFPIVal; - assert((imm >= 0) && (imm <= 0xff)); - fmt = IF_DV_1B; - } - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); - - if (canEncode) - { - imm = fpi.immFPIVal; - assert((imm >= 0) && (imm <= 0xff)); - fmt = IF_DV_1A; - } - } - break; - - default: - // fallback to emit SVE instructions. - return emitInsSve_R_F(ins, attr, reg, immDbl, opt); - - } // end switch (ins) - - assert(canEncode); - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSC(attr, imm); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg); - - dispIns(id); - appendToCurIG(id); -} - -//------------------------------------------------------------------------ -// emitIns_Mov: Emits a move instruction -// -// Arguments: -// ins -- The instruction being emitted -// attr -- The emit attribute -// dstReg -- The destination register -// srcReg -- The source register -// canSkip -- true if the move can be elided when dstReg == srcReg, otherwise false -// insOpts -- The instruction options -// -void emitter::emitIns_Mov( - instruction ins, emitAttr attr, regNumber dstReg, regNumber srcReg, bool canSkip, insOpts opt /* = INS_OPTS_NONE */) -{ - assert(IsMovInstruction(ins)); - - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_mov: - { - assert(insOptsNone(opt)); - - if (IsRedundantMov(ins, size, dstReg, srcReg, canSkip)) - { - // These instructions have no side effect and can be skipped - return; - } - - // Check for the 'mov' aliases for the vector registers - if (isVectorRegister(dstReg)) - { - if (isVectorRegister(srcReg) && isValidVectorDatasize(size)) - { - return emitIns_R_R_R(INS_mov, size, dstReg, srcReg, srcReg); - } - else - { - return emitIns_R_R_I(INS_mov, size, dstReg, srcReg, 0); - } - } - else - { - if (isVectorRegister(srcReg)) - { - assert(isGeneralRegister(dstReg)); - return emitIns_R_R_I(INS_mov, size, dstReg, srcReg, 0); - } - } - - // Is this a MOV to/from SP instruction? - if ((dstReg == REG_SP) || (srcReg == REG_SP)) - { - assert(isGeneralRegisterOrSP(dstReg)); - assert(isGeneralRegisterOrSP(srcReg)); - dstReg = encodingSPtoZR(dstReg); - srcReg = encodingSPtoZR(srcReg); - fmt = IF_DR_2G; - } - else - { - assert(insOptsNone(opt)); - assert(isGeneralRegister(dstReg)); - assert(isGeneralRegisterOrZR(srcReg)); - fmt = IF_DR_2E; - } - break; - } - - case INS_sxtw: - { - assert((size == EA_8BYTE) || (size == EA_4BYTE)); - FALLTHROUGH; - } - - case INS_sxtb: - case INS_sxth: - case INS_uxtb: - case INS_uxth: - { - if (canSkip && (dstReg == srcReg)) - { - // There are scenarios such as in genCallInstruction where the sign/zero extension should be elided - return; - } - - assert(insOptsNone(opt)); - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(dstReg)); - assert(isGeneralRegister(srcReg)); - fmt = IF_DR_2H; - break; - } - - case INS_fmov: - { - assert(isValidVectorElemsizeFloat(size)); - - if (canSkip && (dstReg == srcReg)) - { - // These instructions have no side effect and can be skipped - return; - } - - if (isVectorRegister(dstReg)) - { - if (isVectorRegister(srcReg)) - { - assert(insOptsNone(opt)); - fmt = IF_DV_2G; - } - else - { - assert(isGeneralRegister(srcReg)); - - // if the optional conversion specifier is not present we calculate it - if (opt == INS_OPTS_NONE) - { - opt = (size == EA_4BYTE) ? INS_OPTS_4BYTE_TO_S : INS_OPTS_8BYTE_TO_D; - } - assert(insOptsConvertIntToFloat(opt)); - - fmt = IF_DV_2I; - } - } - else - { - assert(isGeneralRegister(dstReg)); - assert(isVectorRegister(srcReg)); - - // if the optional conversion specifier is not present we calculate it - if (opt == INS_OPTS_NONE) - { - opt = (size == EA_4BYTE) ? INS_OPTS_S_TO_4BYTE : INS_OPTS_D_TO_8BYTE; - } - assert(insOptsConvertFloatToInt(opt)); - - fmt = IF_DV_2H; - } - break; - } - - default: - { - unreached(); - } - } - - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSmall(attr); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(dstReg); - id->idReg2(srcReg); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing two registers - */ - -void emitter::emitIns_R_R(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) -{ - if (IsMovInstruction(ins)) - { - assert(!"Please use emitIns_Mov() to correctly handle move elision"); - emitIns_Mov(ins, attr, reg1, reg2, /* canSkip */ false, opt); - } - - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_dup: - // Vector operation - assert(insOptsAnyArrangement(opt)); - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_2C; - break; - - case INS_abs: - case INS_not: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - if (ins == INS_not) - { - assert(isValidVectorDatasize(size)); - // Bitwise behavior is independent of element size, but is always encoded as 1 Byte - opt = optMakeArrangement(size, EA_1BYTE); - } - if (insOptsNone(opt)) - { - // Scalar operation - assert(size == EA_8BYTE); // Only type D is supported - fmt = IF_DV_2L; - } - else - { - // Vector operation - assert(insOptsAnyArrangement(opt)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - fmt = IF_DV_2M; - } - break; - - case INS_mvn: - case INS_neg: - if (isVectorRegister(reg1)) - { - assert(isVectorRegister(reg2)); - if (ins == INS_mvn) - { - assert(isValidVectorDatasize(size)); - // Bitwise behavior is independent of element size, but is always encoded as 1 Byte - opt = optMakeArrangement(size, EA_1BYTE); - } - if (insOptsNone(opt)) - { - // Scalar operation - assert(size == EA_8BYTE); // Only type D is supported - fmt = IF_DV_2L; - } - else - { - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - fmt = IF_DV_2M; - } - break; - } - FALLTHROUGH; - - case INS_negs: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - fmt = IF_DR_2E; - break; - - case INS_sxtl: - case INS_sxtl2: - case INS_uxtl: - case INS_uxtl2: - return emitIns_R_R_I(ins, size, reg1, reg2, 0, opt); - - case INS_cls: - case INS_clz: - case INS_rbit: - case INS_rev16: - case INS_rev32: - case INS_cnt: - if (isVectorRegister(reg1)) - { - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - if ((ins == INS_cls) || (ins == INS_clz)) - { - assert(elemsize != EA_8BYTE); // No encoding for type D - } - else if (ins == INS_rev32) - { - assert((elemsize == EA_2BYTE) || (elemsize == EA_1BYTE)); - } - else - { - assert(elemsize == EA_1BYTE); // Only supports 8B or 16B - } - fmt = IF_DV_2M; - break; - } - if (ins == INS_cnt) - { - // Doesn't have general register version(s) - break; - } - - FALLTHROUGH; - - case INS_rev: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - if (ins == INS_rev32) - { - assert(size == EA_8BYTE); - } - else - { - assert(isValidGeneralDatasize(size)); - } - fmt = IF_DR_2G; - break; - - case INS_addv: - case INS_saddlv: - case INS_smaxv: - case INS_sminv: - case INS_uaddlv: - case INS_umaxv: - case INS_uminv: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - assert((opt != INS_OPTS_2S) && (opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // Reserved encodings - fmt = IF_DV_2T; - break; - - case INS_rev64: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(elemsize != EA_8BYTE); // No encoding for type D - fmt = IF_DV_2M; - break; - - case INS_sqxtn: - case INS_sqxtun: - case INS_uqxtn: - if (insOptsNone(opt)) - { - // Scalar operation - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorElemsize(size)); - assert(size != EA_8BYTE); // The encoding size = 11 is reserved. - fmt = IF_DV_2L; - break; - } - FALLTHROUGH; - - case INS_xtn: - // Vector operation - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(size == EA_8BYTE); - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = x is reserved - fmt = IF_DV_2M; - break; - - case INS_sqxtn2: - case INS_sqxtun2: - case INS_uqxtn2: - case INS_xtn2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(size == EA_16BYTE); - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_2D); // The encoding size = 11, Q = x is reserved - fmt = IF_DV_2M; - break; - - case INS_ldar: - case INS_ldapr: - case INS_ldaxr: - case INS_ldxr: - case INS_stlr: - assert(isValidGeneralDatasize(size)); - - FALLTHROUGH; - - case INS_ldarb: - case INS_ldaprb: - case INS_ldaxrb: - case INS_ldxrb: - case INS_ldarh: - case INS_ldaprh: - case INS_ldaxrh: - case INS_ldxrh: - case INS_stlrb: - case INS_stlrh: - assert(isValidGeneralLSDatasize(size)); - assert(isGeneralRegisterOrZR(reg1)); - assert(isGeneralRegisterOrSP(reg2)); - assert(insOptsNone(opt)); - - reg2 = encodingSPtoZR(reg2); - - fmt = IF_LS_2A; - break; - - case INS_ldr: - case INS_ldrb: - case INS_ldrh: - case INS_ldrsb: - case INS_ldrsh: - case INS_ldrsw: - case INS_str: - case INS_strb: - case INS_strh: - case INS_cmn: - case INS_tst: - assert(insOptsNone(opt)); - emitIns_R_R_I(ins, attr, reg1, reg2, 0, INS_OPTS_NONE); - return; - - case INS_cmp: - emitIns_R_R_I(ins, attr, reg1, reg2, 0, opt); - return; - - case INS_staddb: - emitIns_R_R_R(INS_ldaddb, attr, reg1, REG_ZR, reg2); - return; - case INS_staddlb: - emitIns_R_R_R(INS_ldaddlb, attr, reg1, REG_ZR, reg2); - return; - case INS_staddh: - emitIns_R_R_R(INS_ldaddh, attr, reg1, REG_ZR, reg2); - return; - case INS_staddlh: - emitIns_R_R_R(INS_ldaddlh, attr, reg1, REG_ZR, reg2); - return; - case INS_stadd: - emitIns_R_R_R(INS_ldadd, attr, reg1, REG_ZR, reg2); - return; - case INS_staddl: - emitIns_R_R_R(INS_ldaddl, attr, reg1, REG_ZR, reg2); - return; - - case INS_fcmp: - case INS_fcmpe: - assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_DV_2K; - break; - - case INS_fcvtns: - case INS_fcvtnu: - case INS_fcvtas: - case INS_fcvtau: - case INS_fcvtps: - case INS_fcvtpu: - case INS_fcvtms: - case INS_fcvtmu: - case INS_fcvtzs: - case INS_fcvtzu: - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsizeFloat(elemsize)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_2A; - } - else - { - // Scalar operation - assert(isVectorRegister(reg2)); - if (isVectorRegister(reg1)) - { - assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); - fmt = IF_DV_2G; - } - else - { - assert(isGeneralRegister(reg1)); - assert(insOptsConvertFloatToInt(opt)); - assert(isValidVectorElemsizeFloat(size)); - fmt = IF_DV_2H; - } - } - break; - - case INS_fcvtl: - case INS_fcvtn: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(size == EA_8BYTE); - assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); - fmt = IF_DV_2A; - break; - - case INS_fcvtl2: - case INS_fcvtn2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(size == EA_16BYTE); - assert((opt == INS_OPTS_8H) || (opt == INS_OPTS_4S)); - fmt = IF_DV_2A; - break; - - case INS_fcvtxn: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(size == EA_8BYTE); - assert(opt == INS_OPTS_2S); - fmt = IF_DV_2A; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(size == EA_4BYTE); - fmt = IF_DV_2G; - } - break; - - case INS_fcvtxn2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(size == EA_16BYTE); - assert(opt == INS_OPTS_4S); - fmt = IF_DV_2A; - break; - - case INS_scvtf: - case INS_ucvtf: - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsizeFloat(elemsize)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_2A; - } - else - { - // Scalar operation - assert(isVectorRegister(reg1)); - if (isVectorRegister(reg2)) - { - assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); - fmt = IF_DV_2G; - } - else - { - assert(isGeneralRegister(reg2)); - assert(insOptsConvertIntToFloat(opt)); - assert(isValidVectorElemsizeFloat(size)); - fmt = IF_DV_2I; - } - } - break; - - case INS_fabs: - case INS_fneg: - case INS_fsqrt: - case INS_frinta: - case INS_frinti: - case INS_frintm: - case INS_frintn: - case INS_frintp: - case INS_frintx: - case INS_frintz: - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsizeFloat(elemsize)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_2A; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(isValidVectorElemsizeFloat(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_DV_2G; - } - break; - - case INS_faddp: - case INS_fmaxnmp: - case INS_fmaxp: - case INS_fminnmp: - case INS_fminp: - // Scalar operation - assert(((size == EA_8BYTE) && (opt == INS_OPTS_2S)) || ((size == EA_16BYTE) && (opt == INS_OPTS_2D))); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_DV_2Q; - break; - - case INS_fmaxnmv: - case INS_fmaxv: - case INS_fminnmv: - case INS_fminv: - assert(size == EA_16BYTE); - assert(opt == INS_OPTS_4S); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_DV_2R; - break; - - case INS_addp: - assert(size == EA_16BYTE); - assert(opt == INS_OPTS_2D); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_DV_2S; - break; - - case INS_fcvt: - assert(insOptsConvertFloatToFloat(opt)); - assert(isValidVectorFcvtsize(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_DV_2J; - break; - - case INS_cmeq: - case INS_cmge: - case INS_cmgt: - case INS_cmle: - case INS_cmlt: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - fmt = IF_DV_2M; - } - else - { - // Scalar operation - assert(size == EA_8BYTE); - assert(insOptsNone(opt)); - fmt = IF_DV_2L; - } - break; - - case INS_fcmeq: - case INS_fcmge: - case INS_fcmgt: - case INS_fcmle: - case INS_fcmlt: - case INS_frecpe: - case INS_frsqrte: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsizeFloat(elemsize)); // Only Double/Float supported - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_2A; - } - else - { - // Scalar operation - assert(isValidScalarDatasize(size)); // Only Double/Float supported - assert(insOptsNone(opt)); - fmt = IF_DV_2G; - } - break; - - case INS_aesd: - case INS_aese: - case INS_aesmc: - case INS_aesimc: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - elemsize = optGetElemsize(opt); - assert(elemsize == EA_1BYTE); - fmt = IF_DV_2P; - break; - - case INS_sha1h: - assert(insOptsNone(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_DV_2U; - break; - - case INS_sha256su0: - case INS_sha1su1: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - elemsize = optGetElemsize(opt); - assert(elemsize == EA_4BYTE); - fmt = IF_DV_2P; - break; - - case INS_ld2: - case INS_ld3: - case INS_ld4: - case INS_st2: - case INS_st3: - case INS_st4: - assert(opt != INS_OPTS_1D); // .1D format only permitted with LD1 & ST1 - FALLTHROUGH; - - case INS_ld1: - case INS_ld1_2regs: - case INS_ld1_3regs: - case INS_ld1_4regs: - case INS_st1: - case INS_st1_2regs: - case INS_st1_3regs: - case INS_st1_4regs: - case INS_ld1r: - case INS_ld2r: - case INS_ld3r: - case INS_ld4r: - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrSP(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - - // Load/Store multiple structures base register - // Load single structure and replicate base register - reg2 = encodingSPtoZR(reg2); - fmt = IF_LS_2D; - break; - - case INS_urecpe: - case INS_ursqrte: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(elemsize == EA_4BYTE); - fmt = IF_DV_2A; - break; - - case INS_frecpx: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidScalarDatasize(size)); - assert(insOptsNone(opt)); - fmt = IF_DV_2G; - break; - - case INS_sadalp: - case INS_saddlp: - case INS_uadalp: - case INS_uaddlp: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidArrangement(size, opt)); - assert((opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // The encoding size = 11, Q = x is reserved - fmt = IF_DV_2T; - break; - - case INS_sqabs: - case INS_sqneg: - case INS_suqadd: - case INS_usqadd: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = 0 is reserved - fmt = IF_DV_2M; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(isValidVectorElemsize(size)); - fmt = IF_DV_2L; - } - break; - - default: - // fallback to emit SVE instructions. - return emitInsSve_R_R(ins, attr, reg1, reg2, opt, sopt); - - } // end switch (ins) - - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSmall(attr); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg1); - id->idReg2(reg2); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing a register and two constants. - */ - -void emitter::emitIns_R_I_I(instruction ins, - emitAttr attr, - regNumber reg, - ssize_t imm1, - ssize_t imm2, - insOpts opt /* = INS_OPTS_NONE */ - DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = 0 */)) -{ - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - size_t immOut = 0; // composed from imm1 and imm2 and stored in the instrDesc - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - bool canEncode; - halfwordImm hwi; - - case INS_mov: - ins = INS_movz; // INS_mov with LSL is an alias for INS_movz LSL - FALLTHROUGH; - - case INS_movk: - case INS_movn: - case INS_movz: - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg)); - assert(isValidUimm<16>(imm1)); - assert(insOptsLSL(opt)); // Must be INS_OPTS_LSL - - if (size == EA_8BYTE) - { - assert((imm2 == 0) || (imm2 == 16) || // shift amount: 0, 16, 32 or 48 - (imm2 == 32) || (imm2 == 48)); - } - else // EA_4BYTE - { - assert((imm2 == 0) || (imm2 == 16)); // shift amount: 0 or 16 - } - - hwi.immHWVal = 0; - - switch (imm2) - { - case 0: - hwi.immHW = 0; - canEncode = true; - break; - - case 16: - hwi.immHW = 1; - canEncode = true; - break; - - case 32: - hwi.immHW = 2; - canEncode = true; - break; - - case 48: - hwi.immHW = 3; - canEncode = true; - break; - - default: - canEncode = false; - } - - if (canEncode) - { - hwi.immVal = imm1; - - immOut = hwi.immHWVal; - assert(isValidImmHWVal(immOut, size)); - fmt = IF_DI_1B; - } - break; - - default: - // fallback to emit SVE instructions. - return emitInsSve_R_I_I(ins, attr, reg, imm1, imm2, opt); - - } // end switch (ins) - - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSC(attr, immOut); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg); - -#ifdef DEBUG - id->idDebugOnlyInfo()->idFlags = gtFlags; - id->idDebugOnlyInfo()->idMemCookie = targetHandle; -#endif - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing two registers and a constant. - */ - -void emitter::emitIns_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) -{ - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - bool isLdSt = false; - bool isLdrStr = false; - bool isSIMD = false; - bool isAddSub = false; - bool setFlags = false; - unsigned scale = 0; - bool unscaledOp = false; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - bool canEncode; - bitMaskImm bmi; - unsigned registerListSize; - bool isRightShift; - - case INS_mov: - // Check for the 'mov' aliases for the vector registers - assert(insOptsNone(opt)); - assert(isValidVectorElemsize(size)); - elemsize = size; - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - - if (isVectorRegister(reg1)) - { - if (isGeneralRegisterOrZR(reg2)) - { - fmt = IF_DV_2C; // Alias for 'ins' - break; - } - else if (isVectorRegister(reg2)) - { - fmt = IF_DV_2E; // Alias for 'dup' - break; - } - } - else // isGeneralRegister(reg1) - { - assert(isGeneralRegister(reg1)); - if (isVectorRegister(reg2)) - { - fmt = IF_DV_2B; // Alias for 'umov' - break; - } - } - assert(!" invalid INS_mov operands"); - break; - - case INS_lsl: - case INS_lsr: - case INS_asr: - assert(insOptsNone(opt)); - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isValidImmShift(imm, size)); - fmt = IF_DI_2D; - break; - - case INS_ror: - assert(insOptsNone(opt)); - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isValidImmShift(imm, size)); - fmt = IF_DI_2B; - break; - - case INS_shl: - case INS_sli: - case INS_sri: - case INS_srshr: - case INS_srsra: - case INS_sshr: - case INS_ssra: - case INS_urshr: - case INS_ursra: - case INS_ushr: - case INS_usra: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - isRightShift = emitInsIsVectorRightShift(ins); - - assert(!isRightShift || - (imm != 0 && "instructions for vector right-shift do not allow zero as an immediate value")); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsize(elemsize)); - assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_2O; - break; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(size == EA_8BYTE); // only supported size - assert(isValidVectorShiftAmount(imm, size, isRightShift)); - fmt = IF_DV_2N; - } - break; - - case INS_sqshl: - case INS_uqshl: - case INS_sqshlu: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - isRightShift = emitInsIsVectorRightShift(ins); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_1D); // The encoding immh = 1xxx, Q = 0 is reserved - elemsize = optGetElemsize(opt); - assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); - fmt = IF_DV_2O; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(isValidVectorElemsize(size)); - assert(isValidVectorShiftAmount(imm, size, isRightShift)); - fmt = IF_DV_2N; - } - break; - - case INS_sqrshrn: - case INS_sqrshrun: - case INS_sqshrn: - case INS_sqshrun: - case INS_uqrshrn: - case INS_uqshrn: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - isRightShift = emitInsIsVectorRightShift(ins); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidArrangement(size, opt)); - assert((opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // The encoding immh = 1xxx, Q = x is reserved - elemsize = optGetElemsize(opt); - assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); - fmt = IF_DV_2O; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(isValidVectorElemsize(size)); - assert(size != EA_8BYTE); // The encoding immh = 1xxx is reserved - assert(isValidVectorShiftAmount(imm, size, isRightShift)); - fmt = IF_DV_2N; - } - break; - - case INS_sxtl: - case INS_uxtl: - assert(imm == 0); - FALLTHROUGH; - - case INS_rshrn: - case INS_shrn: - case INS_sshll: - case INS_ushll: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - isRightShift = emitInsIsVectorRightShift(ins); - // Vector operation - assert(size == EA_8BYTE); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(elemsize != EA_8BYTE); // Reserved encodings - assert(isValidVectorElemsize(elemsize)); - assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); - fmt = IF_DV_2O; - break; - - case INS_sxtl2: - case INS_uxtl2: - assert(imm == 0); - FALLTHROUGH; - - case INS_rshrn2: - case INS_shrn2: - case INS_sqrshrn2: - case INS_sqrshrun2: - case INS_sqshrn2: - case INS_sqshrun2: - case INS_sshll2: - case INS_uqrshrn2: - case INS_uqshrn2: - case INS_ushll2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - isRightShift = emitInsIsVectorRightShift(ins); - - // Vector operation - assert(size == EA_16BYTE); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(elemsize != EA_8BYTE); // The encoding immh = 1xxx, Q = x is reserved - assert(isValidVectorElemsize(elemsize)); - assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); - fmt = IF_DV_2O; - break; - - case INS_mvn: - case INS_neg: - case INS_negs: - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - - if (imm == 0) - { - assert(insOptsNone(opt)); // a zero imm, means no alu shift kind - - fmt = IF_DR_2E; - } - else - { - if (ins == INS_mvn) - { - assert(insOptsAnyShift(opt)); // a non-zero imm, must select shift kind - } - else // neg or negs - { - assert(insOptsAluShift(opt)); // a non-zero imm, must select shift kind, can't use ROR - } - assert(isValidImmShift(imm, size)); - fmt = IF_DR_2F; - } - break; - - case INS_tst: - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegisterOrZR(reg1)); - assert(isGeneralRegister(reg2)); - - if (insOptsAnyShift(opt)) - { - assert(isValidImmShift(imm, size) && (imm != 0)); - fmt = IF_DR_2B; - } - else - { - assert(insOptsNone(opt)); // a zero imm, means no alu shift kind - assert(imm == 0); - fmt = IF_DR_2A; - } - break; - - case INS_cmp: - case INS_cmn: - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegisterOrSP(reg1)); - assert(isGeneralRegister(reg2)); - - reg1 = encodingSPtoZR(reg1); - if (insOptsAnyExtend(opt)) - { - assert((imm >= 0) && (imm <= 4)); - - fmt = IF_DR_2C; - } - else if (imm == 0) - { - assert(insOptsNone(opt)); // a zero imm, means no alu shift kind - - fmt = IF_DR_2A; - } - else - { - assert(insOptsAnyShift(opt)); // a non-zero imm, must select shift kind - assert(isValidImmShift(imm, size)); - fmt = IF_DR_2B; - } - break; - - case INS_ands: - case INS_and: - case INS_eor: - case INS_orr: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg2)); - if (ins == INS_ands) - { - assert(isGeneralRegister(reg1)); - } - else - { - assert(isGeneralRegisterOrSP(reg1)); - reg1 = encodingSPtoZR(reg1); - } - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, size, &bmi); - if (canEncode) - { - imm = bmi.immNRS; - assert(isValidImmNRS(imm, size)); - fmt = IF_DI_2C; - } - break; - - case INS_dup: // by element, imm selects the element of reg2 - assert(isVectorRegister(reg1)); - if (isVectorRegister(reg2)) - { - if (insOptsAnyArrangement(opt)) - { - // The size and opt were modified to be based on the - // return type but the immediate is based on the operand - // which can be of a larger size. As such, we don't - // assert the index is valid here and instead do it in - // codegen. - - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsize(elemsize)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_2D; - break; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - elemsize = size; - assert(isValidVectorElemsize(elemsize)); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - fmt = IF_DV_2E; - break; - } - } - FALLTHROUGH; - - case INS_ins: // (MOV from general) - assert(insOptsNone(opt)); - assert(isValidVectorElemsize(size)); - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - elemsize = size; - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - fmt = IF_DV_2C; - break; - - case INS_umov: // (MOV to general) - assert(insOptsNone(opt)); - assert(isValidVectorElemsize(size)); - assert(isGeneralRegister(reg1)); - assert(isVectorRegister(reg2)); - elemsize = size; - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - fmt = IF_DV_2B; - break; - - case INS_smov: - assert(insOptsNone(opt)); - assert(isValidVectorElemsize(size)); - assert(size != EA_8BYTE); // no encoding, use INS_umov - assert(isGeneralRegister(reg1)); - assert(isVectorRegister(reg2)); - elemsize = size; - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - fmt = IF_DV_2B; - break; - - case INS_add: - case INS_sub: - setFlags = false; - isAddSub = true; - break; - - case INS_adds: - case INS_subs: - setFlags = true; - isAddSub = true; - break; - - case INS_ldrsb: - case INS_ldursb: - // 'size' specifies how we sign-extend into 4 or 8 bytes of the target register - assert(isValidGeneralDatasize(size)); - unscaledOp = (ins == INS_ldursb); - scale = 0; - isLdSt = true; - break; - - case INS_ldrsh: - case INS_ldursh: - // 'size' specifies how we sign-extend into 4 or 8 bytes of the target register - assert(isValidGeneralDatasize(size)); - unscaledOp = (ins == INS_ldursh); - scale = 1; - isLdSt = true; - break; - - case INS_ldrsw: - case INS_ldursw: - // 'size' specifies how we sign-extend into 4 or 8 bytes of the target register - assert(size == EA_8BYTE); - unscaledOp = (ins == INS_ldursw); - scale = 2; - isLdSt = true; - break; - - case INS_ldrb: - case INS_strb: - // size is ignored - unscaledOp = false; - scale = 0; - isLdSt = true; - break; - - case INS_ldapurb: - case INS_stlurb: - case INS_ldurb: - case INS_sturb: - // size is ignored - unscaledOp = true; - scale = 0; - isLdSt = true; - break; - - case INS_ldrh: - case INS_strh: - // size is ignored - unscaledOp = false; - scale = 1; - isLdSt = true; - break; - - case INS_ldurh: - case INS_ldapurh: - case INS_sturh: - case INS_stlurh: - // size is ignored - unscaledOp = true; - scale = 0; - isLdSt = true; - break; - - case INS_ldr: - case INS_str: - // Is the target a vector register? - if (isVectorRegister(reg1)) - { - assert(isValidVectorLSDatasize(size)); - assert(isGeneralRegisterOrSP(reg2)); - isSIMD = true; - } - else - { - assert(isValidGeneralDatasize(size)); - } - unscaledOp = false; - scale = NaturalScale_helper(size); - isLdSt = true; - isLdrStr = true; - break; - - case INS_ldur: - case INS_stur: - case INS_ldapur: - case INS_stlur: - // Is the target a vector register? - if (isVectorRegister(reg1)) - { - assert(isValidVectorLSDatasize(size)); - assert(isGeneralRegisterOrSP(reg2)); - isSIMD = true; - } - else - { - assert(isValidGeneralDatasize(size)); - } - unscaledOp = true; - scale = 0; - isLdSt = true; - break; - - case INS_ld2: - case INS_ld3: - case INS_ld4: - case INS_st2: - case INS_st3: - case INS_st4: - assert(opt != INS_OPTS_1D); // .1D format only permitted with LD1 & ST1 - FALLTHROUGH; - - case INS_ld1: - case INS_ld1_2regs: - case INS_ld1_3regs: - case INS_ld1_4regs: - case INS_st1: - case INS_st1_2regs: - case INS_st1_3regs: - case INS_st1_4regs: - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrSP(reg2)); - - reg2 = encodingSPtoZR(reg2); - - if (insOptsAnyArrangement(opt)) - { - registerListSize = insGetRegisterListSize(ins); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - assert((size * registerListSize) == imm); - - // Load/Store multiple structures post-indexed by an immediate - fmt = IF_LS_2E; - } - else - { - assert(insOptsNone(opt)); - assert((ins != INS_ld1_2regs) && (ins != INS_ld1_3regs) && (ins != INS_ld1_4regs) && - (ins != INS_st1_2regs) && (ins != INS_st1_3regs) && (ins != INS_st1_4regs)); - - elemsize = size; - assert(isValidVectorElemsize(elemsize)); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - - // Load/Store single structure base register - fmt = IF_LS_2F; - } - break; - - case INS_ld1r: - case INS_ld2r: - case INS_ld3r: - case INS_ld4r: - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrSP(reg2)); - - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - - elemsize = optGetElemsize(opt); - registerListSize = insGetRegisterListSize(ins); - assert((elemsize * registerListSize) == imm); - - // Load single structure and replicate post-indexed by an immediate - reg2 = encodingSPtoZR(reg2); - fmt = IF_LS_2E; - break; - - default: - // fallback to emit SVE instructions. - return emitInsSve_R_R_I(ins, attr, reg1, reg2, imm, opt, sopt); - - } // end switch (ins) - - if (isLdSt) - { - assert(!isAddSub); - - if (isSIMD) - { - assert(isValidVectorLSDatasize(size)); - assert(isVectorRegister(reg1)); - assert((scale >= 0) && (scale <= 4)); - } - else - { - assert(isValidGeneralLSDatasize(size)); - assert(isGeneralRegisterOrZR(reg1)); - assert((scale >= 0) && (scale <= 3)); - } - - assert(isGeneralRegisterOrSP(reg2)); - - // Load/Store reserved encodings: - if (insOptsIndexed(opt)) - { - assert(reg1 != reg2); - } - - reg2 = encodingSPtoZR(reg2); - - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - if (imm == 0 || EA_IS_CNS_TLSGD_RELOC(attr)) - { - assert(insOptsNone(opt)); // PRE/POST Index doesn't make sense with an immediate of zero - - fmt = IF_LS_2A; - } - else if (insOptsIndexed(opt) || unscaledOp || (imm < 0) || ((imm & mask) != 0)) - { - if ((imm >= -256) && (imm <= 255)) - { - fmt = IF_LS_2C; - } - else - { - assert(!"Instruction cannot be encoded: IF_LS_2C"); - } - } - else if (imm > 0) - { - assert(insOptsNone(opt)); - assert(!unscaledOp); - - if (((imm & mask) == 0) && ((imm >> scale) < 0x1000)) - { - imm >>= scale; // The immediate is scaled by the size of the ld/st - - fmt = IF_LS_2B; - } - else - { - assert(!"Instruction cannot be encoded: IF_LS_2B"); - } - } - - // Try to optimize a load/store with an alternative instruction. - if (isLdrStr && emitComp->opts.OptimizationEnabled() && - OptimizeLdrStr(ins, attr, reg1, reg2, imm, size, fmt, false, -1, -1 DEBUG_ARG(false))) - { - return; - } - } - else if (isAddSub) - { - assert(!isLdSt); - assert(insOptsNone(opt)); - - if (setFlags) // Can't encode SP with setFlags - { - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - } - else - { - assert(isGeneralRegisterOrSP(reg1)); - assert(isGeneralRegisterOrSP(reg2)); - - // Is it just a mov? - if (imm == 0) - { - emitIns_Mov(INS_mov, attr, reg1, reg2, /* canSkip */ true); - return; - } - - reg1 = encodingSPtoZR(reg1); - reg2 = encodingSPtoZR(reg2); - } - - if (unsigned_abs(imm) <= 0x0fff) - { - if (imm < 0) - { - ins = insReverse(ins); - imm = -imm; - } - assert(isValidUimm<12>(imm)); - fmt = IF_DI_2A; - } - else if (canEncodeWithShiftImmBy12(imm)) // Try the shifted by 12 encoding - { - // Encoding will use a 12-bit left shift of the immediate - opt = INS_OPTS_LSL12; - if (imm < 0) - { - ins = insReverse(ins); - imm = -imm; - } - assert((imm & 0xfff) == 0); - imm >>= 12; - assert(isValidUimm<12>(imm)); - fmt = IF_DI_2A; - } - else - { - assert(!"Instruction cannot be encoded: IF_DI_2A"); - } - } - - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSC(attr, imm); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg1); - id->idReg2(reg2); - - if (EA_IS_CNS_TLSGD_RELOC(attr)) - { - assert(imm != 0); - id->idSetTlsGD(); - } - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing two registers and a floating point constant. - */ - -void emitter::emitIns_R_R_F( - instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, double immDbl, insOpts opt /* = INS_OPTS_NONE */) -{ - // Currently, only SVE instructions use this format. - emitInsSve_R_R_F(ins, attr, reg1, reg2, immDbl, opt); -} - -/***************************************************************************** -* -* Add an instruction referencing two registers and a constant. -* Also checks for a large immediate that needs a second instruction -* and will load it in reg1 -* -* - Supports instructions: add, adds, sub, subs, and, ands, eor and orr -* - Requires that reg1 is a general register and not SP or ZR -* - Requires that reg1 != reg2 -*/ -void emitter::emitIns_R_R_Imm(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, ssize_t imm) -{ - assert(isGeneralRegister(reg1)); - assert(reg1 != reg2); - - bool immFits = true; - - switch (ins) - { - case INS_add: - case INS_adds: - case INS_sub: - case INS_subs: - immFits = emitter::emitIns_valid_imm_for_add(imm, attr); - break; - - case INS_ands: - case INS_and: - case INS_eor: - case INS_orr: - immFits = emitter::emitIns_valid_imm_for_alu(imm, attr); - break; - - default: - assert(!"Unsupported instruction in emitIns_R_R_Imm"); - } - - if (immFits) - { - emitIns_R_R_I(ins, attr, reg1, reg2, imm); - } - else - { - // Load 'imm' into the reg1 register - // then issue: 'ins' reg1, reg2, reg1 - // - codeGen->instGen_Set_Reg_To_Imm(attr, reg1, imm); - emitIns_R_R_R(ins, attr, reg1, reg2, reg1); - } -} - -/***************************************************************************** - * - * Add an instruction referencing three registers. - */ - -void emitter::emitIns_R_R_R(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) -{ - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_mul: - case INS_smull: - case INS_umull: - if (insOptsAnyArrangement(opt)) - { - // ASIMD instruction - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isValidArrangement(size, opt)); - assert((opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // The encoding size = 11, Q = x is reserved - fmt = IF_DV_3A; - break; - } - // Base instruction - FALLTHROUGH; - - case INS_lsl: - case INS_lsr: - case INS_asr: - case INS_ror: - case INS_adc: - case INS_adcs: - case INS_sbc: - case INS_sbcs: - case INS_udiv: - case INS_sdiv: - case INS_mneg: - case INS_smnegl: - case INS_smulh: - case INS_umnegl: - case INS_umulh: - case INS_lslv: - case INS_lsrv: - case INS_asrv: - case INS_rorv: - case INS_crc32b: - case INS_crc32h: - case INS_crc32w: - case INS_crc32x: - case INS_crc32cb: - case INS_crc32ch: - case INS_crc32cw: - case INS_crc32cx: - assert(insOptsNone(opt)); - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isGeneralRegister(reg3)); - fmt = IF_DR_3A; - break; - - case INS_add: - case INS_sub: - if (isVectorRegister(reg1)) - { - // ASIMD instruction - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(opt != INS_OPTS_1D); // Reserved encoding - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - fmt = IF_DV_3A; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(size == EA_8BYTE); - fmt = IF_DV_3E; - } - break; - } - // Base instruction - FALLTHROUGH; - - case INS_adds: - case INS_subs: - emitIns_R_R_R_I(ins, attr, reg1, reg2, reg3, 0, opt); - return; - - case INS_cmeq: - case INS_cmge: - case INS_cmgt: - case INS_cmhi: - case INS_cmhs: - case INS_cmtst: - case INS_srshl: - case INS_sshl: - case INS_urshl: - case INS_ushl: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = 0 is reserved - fmt = IF_DV_3A; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(size == EA_8BYTE); // Only Int64/UInt64 supported - fmt = IF_DV_3E; - } - break; - - case INS_sqadd: - case INS_sqrshl: - case INS_sqshl: - case INS_sqsub: - case INS_uqadd: - case INS_uqrshl: - case INS_uqshl: - case INS_uqsub: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = 0 is reserved - fmt = IF_DV_3A; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(isValidVectorElemsize(size)); - fmt = IF_DV_3E; - } - break; - - case INS_fcmeq: - case INS_fcmge: - case INS_fcmgt: - case INS_frecps: - case INS_frsqrts: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert((elemsize == EA_8BYTE) || (elemsize == EA_4BYTE)); // Only Double/Float supported - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_3B; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert((size == EA_8BYTE) || (size == EA_4BYTE)); // Only Double/Float supported - fmt = IF_DV_3D; - } - break; - - case INS_mla: - case INS_mls: - case INS_saba: - case INS_sabd: - case INS_shadd: - case INS_shsub: - case INS_smax: - case INS_smaxp: - case INS_smin: - case INS_sminp: - case INS_srhadd: - case INS_uaba: - case INS_uabd: - case INS_uhadd: - case INS_uhsub: - case INS_umax: - case INS_umaxp: - case INS_umin: - case INS_uminp: - case INS_urhadd: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isValidArrangement(size, opt)); - assert((opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // The encoding size = 11, Q = x is reserved - fmt = IF_DV_3A; - break; - - case INS_addp: - case INS_uzp1: - case INS_uzp2: - case INS_zip1: - case INS_zip2: - case INS_trn1: - case INS_trn2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = 0 is reserved - fmt = IF_DV_3A; - break; - - case INS_mov: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(reg2 == reg3); - assert(isValidVectorDatasize(size)); - // INS_mov is an alias for INS_orr (vector register) - if (opt == INS_OPTS_NONE) - { - elemsize = EA_1BYTE; - opt = optMakeArrangement(size, elemsize); - } - assert(isValidArrangement(size, opt)); - fmt = IF_DV_3C; - break; - - case INS_and: - case INS_bic: - case INS_eor: - case INS_orr: - case INS_orn: - case INS_tbl: - case INS_tbl_2regs: - case INS_tbl_3regs: - case INS_tbl_4regs: - case INS_tbx: - case INS_tbx_2regs: - case INS_tbx_3regs: - case INS_tbx_4regs: - if (isVectorRegister(reg1)) - { - assert(isValidVectorDatasize(size)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (opt == INS_OPTS_NONE) - { - elemsize = EA_1BYTE; - opt = optMakeArrangement(size, elemsize); - } - assert(isValidArrangement(size, opt)); - fmt = IF_DV_3C; - break; - } - FALLTHROUGH; - - case INS_ands: - case INS_bics: - case INS_eon: - emitIns_R_R_R_I(ins, attr, reg1, reg2, reg3, 0, INS_OPTS_NONE); - return; - - case INS_bsl: - case INS_bit: - case INS_bif: - assert(isValidVectorDatasize(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (opt == INS_OPTS_NONE) - { - elemsize = EA_1BYTE; - opt = optMakeArrangement(size, elemsize); - } - assert(isValidArrangement(size, opt)); - fmt = IF_DV_3C; - break; - - case INS_fadd: - case INS_fsub: - case INS_fdiv: - case INS_fmax: - case INS_fmaxnm: - case INS_fmin: - case INS_fminnm: - case INS_fabd: - case INS_fmul: - case INS_fmulx: - case INS_facge: - case INS_facgt: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsizeFloat(elemsize)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_3B; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(isValidScalarDatasize(size)); - fmt = IF_DV_3D; - } - break; - - case INS_fnmul: - // Scalar operation - assert(insOptsNone(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isValidScalarDatasize(size)); - fmt = IF_DV_3D; - break; - - case INS_faddp: - case INS_fmaxnmp: - case INS_fmaxp: - case INS_fminnmp: - case INS_fminp: - - case INS_fmla: - case INS_fmls: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsAnyArrangement(opt)); // no scalar encoding, use 4-operand 'fmadd' or 'fmsub' - - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsizeFloat(elemsize)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_3B; - break; - - case INS_ldr: - case INS_ldrb: - case INS_ldrh: - case INS_ldrsb: - case INS_ldrsh: - case INS_ldrsw: - case INS_str: - case INS_strb: - case INS_strh: - emitIns_R_R_R_Ext(ins, attr, reg1, reg2, reg3, opt); - return; - - case INS_ldp: - case INS_ldpsw: - case INS_ldnp: - case INS_stp: - case INS_stnp: - emitIns_R_R_R_I(ins, attr, reg1, reg2, reg3, 0); - return; - - case INS_stxr: - case INS_stxrb: - case INS_stxrh: - case INS_stlxr: - case INS_stlxrb: - case INS_stlxrh: - assert(isGeneralRegisterOrZR(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert(isGeneralRegisterOrSP(reg3)); - fmt = IF_LS_3D; - break; - - case INS_casb: - case INS_casab: - case INS_casalb: - case INS_caslb: - case INS_cash: - case INS_casah: - case INS_casalh: - case INS_caslh: - case INS_cas: - case INS_casa: - case INS_casal: - case INS_casl: - case INS_ldaddb: - case INS_ldaddab: - case INS_ldaddalb: - case INS_ldaddlb: - case INS_ldaddh: - case INS_ldaddah: - case INS_ldaddalh: - case INS_ldaddlh: - case INS_ldadd: - case INS_ldadda: - case INS_ldaddal: - case INS_ldaddl: - case INS_ldclral: - case INS_ldsetal: - case INS_swpb: - case INS_swpab: - case INS_swpalb: - case INS_swplb: - case INS_swph: - case INS_swpah: - case INS_swpalh: - case INS_swplh: - case INS_swp: - case INS_swpa: - case INS_swpal: - case INS_swpl: - assert(isGeneralRegisterOrZR(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert(isGeneralRegisterOrSP(reg3)); - fmt = IF_LS_3E; - break; - - case INS_sha256h: - case INS_sha256h2: - case INS_sha256su1: - case INS_sha1su0: - case INS_sha1c: - case INS_sha1p: - case INS_sha1m: - assert(isValidVectorDatasize(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (opt == INS_OPTS_NONE) - { - elemsize = EA_4BYTE; - opt = optMakeArrangement(size, elemsize); - } - assert(isValidArrangement(size, opt)); - fmt = IF_DV_3F; - break; - - case INS_ld2: - case INS_ld3: - case INS_ld4: - case INS_st2: - case INS_st3: - case INS_st4: - assert(opt != INS_OPTS_1D); // .1D format only permitted with LD1 & ST1 - FALLTHROUGH; - - case INS_ld1: - case INS_ld1_2regs: - case INS_ld1_3regs: - case INS_ld1_4regs: - case INS_st1: - case INS_st1_2regs: - case INS_st1_3regs: - case INS_st1_4regs: - case INS_ld1r: - case INS_ld2r: - case INS_ld3r: - case INS_ld4r: - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrSP(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidArrangement(size, opt)); - - // Load/Store multiple structures post-indexed by a register - // Load single structure and replicate post-indexed by a register - reg2 = encodingSPtoZR(reg2); - fmt = IF_LS_3F; - break; - - case INS_addhn: - case INS_raddhn: - case INS_rsubhn: - case INS_subhn: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_8BYTE); - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = x is reserved. - fmt = IF_DV_3A; - break; - - case INS_addhn2: - case INS_raddhn2: - case INS_rsubhn2: - case INS_subhn2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_16BYTE); - assert(isValidArrangement(size, opt)); - assert(opt != INS_OPTS_2D); // The encoding size = 11, Q = x is reserved. - fmt = IF_DV_3A; - break; - - case INS_sabal: - case INS_sabdl: - case INS_saddl: - case INS_saddw: - case INS_smlal: - case INS_smlsl: - case INS_ssubl: - case INS_ssubw: - case INS_uabal: - case INS_uabdl: - case INS_uaddl: - case INS_uaddw: - case INS_umlal: - case INS_umlsl: - case INS_usubl: - case INS_usubw: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_8BYTE); - assert((opt == INS_OPTS_8B) || (opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); - fmt = IF_DV_3A; - break; - - case INS_sabal2: - case INS_sabdl2: - case INS_saddl2: - case INS_saddw2: - case INS_smlal2: - case INS_smlsl2: - case INS_ssubl2: - case INS_ssubw2: - case INS_umlal2: - case INS_umlsl2: - case INS_smull2: - case INS_uabal2: - case INS_uabdl2: - case INS_uaddl2: - case INS_uaddw2: - case INS_usubl2: - case INS_umull2: - case INS_usubw2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_16BYTE); - assert((opt == INS_OPTS_16B) || (opt == INS_OPTS_8H) || (opt == INS_OPTS_4S)); - fmt = IF_DV_3A; - break; - - case INS_sqdmlal: - case INS_sqdmlsl: - case INS_sqdmull: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(size == EA_8BYTE); - assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); - fmt = IF_DV_3A; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert((size == EA_2BYTE) || (size == EA_4BYTE)); - fmt = IF_DV_3E; - } - break; - - case INS_sqdmulh: - case INS_sqrdmlah: - case INS_sqrdmlsh: - case INS_sqrdmulh: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - elemsize = optGetElemsize(opt); - assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)); - fmt = IF_DV_3A; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert((size == EA_2BYTE) || (size == EA_4BYTE)); - fmt = IF_DV_3E; - } - break; - - case INS_sqdmlal2: - case INS_sqdmlsl2: - case INS_sqdmull2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_16BYTE); - assert((opt == INS_OPTS_8H) || (opt == INS_OPTS_4S)); - fmt = IF_DV_3A; - break; - - case INS_pmul: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isValidArrangement(size, opt)); - assert((opt == INS_OPTS_8B) || (opt == INS_OPTS_16B)); - fmt = IF_DV_3A; - break; - - case INS_pmull: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_8BYTE); - assert((opt == INS_OPTS_8B) || (opt == INS_OPTS_1D)); - fmt = IF_DV_3A; - break; - - case INS_pmull2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_16BYTE); - assert((opt == INS_OPTS_16B) || (opt == INS_OPTS_2D)); - fmt = IF_DV_3A; - break; - - case INS_sdot: - case INS_udot: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(((size == EA_8BYTE) && (opt == INS_OPTS_2S)) || ((size == EA_16BYTE) && (opt == INS_OPTS_4S))); - fmt = IF_DV_3A; - break; - - default: - // fallback to emit SVE instructions. - return emitInsSve_R_R_R(ins, attr, reg1, reg2, reg3, opt, sopt); - - } // end switch (ins) - - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstr(attr); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - - dispIns(id); - appendToCurIG(id); + // mask off any extra bits that we got from the left shift + result &= ((1ULL << width) - 1); + } + return result; } +/************************************************************************ + * + * A helper method to perform a 'NOT' bitwise complement operation. + * 'value' is considered to be a fixed size 'width' set of bits. + * + * Example + * value is '01001011', and width is 8 + * result is '10110100' + */ -//----------------------------------------------------------------------------------- -// emitIns_R_R_R_I_LdStPair: Add an instruction storing 2 registers into a memory -// (pointed by reg3) and the offset (immediate). -// -// Arguments: -// ins - The instruction code -// attr - The emit attribute for register 1 -// attr2 - The emit attribute for register 2 -// reg1 - Register 1 -// reg2 - Register 2 -// reg3 - Register 3 -// imm - Immediate offset, prior to scaling by operand size -// varx1 - LclVar number 1 -// varx2 - LclVar number 2 -// offs1 - Memory offset of lclvar number 1 -// offs2 - Memory offset of lclvar number 2 -// -void emitter::emitIns_R_R_R_I_LdStPair(instruction ins, - emitAttr attr, - emitAttr attr2, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - int varx1, - int varx2, - int offs1, - int offs2 DEBUG_ARG(unsigned var1RefsOffs) DEBUG_ARG(unsigned var2RefsOffs)) +/*static*/ UINT64 emitter::NOT_helper(UINT64 value, unsigned width) { - assert((ins == INS_stp) || (ins == INS_ldp)); - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - unsigned scale = 0; + assert(width <= 64); - // Is the target a vector register? - if (isVectorRegister(reg1)) - { - assert(isValidVectorLSPDatasize(size)); - assert(isVectorRegister(reg2)); + UINT64 result = ~value; - scale = NaturalScale_helper(size); - assert((scale >= 2) && (scale <= 4)); - } - else + if (width < 64) { - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegisterOrZR(reg2)); - scale = (size == EA_8BYTE) ? 3 : 2; - } - - reg3 = encodingSPtoZR(reg3); + // Check that 'value' fits in 'width' bits. Don't consider "sign" bits above width. + UINT64 maxVal = 1ULL << width; + UINT64 lowBitsMask = maxVal - 1; + UINT64 signBitsMask = ~lowBitsMask | (1ULL << (width - 1)); // The high bits must be set, and the top bit + // (sign bit) must be set. + assert((value < maxVal) || ((value & signBitsMask) == signBitsMask)); - fmt = IF_LS_3C; - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - if (imm == 0) - { - fmt = IF_LS_3B; - } - else - { - if ((imm & mask) == 0) - { - imm >>= scale; // The immediate is scaled by the size of the ld/st - } - else - { - // Unlike emitIns_S_S_R_R(), we would never come here when - // (imm & mask) != 0. - unreached(); - } + // mask off any extra bits that we got from the complement operation + result &= lowBitsMask; } - bool validVar1 = varx1 != -1; - bool validVar2 = varx2 != -1; + return result; +} - instrDesc* id; +/************************************************************************ + * + * A helper method to perform a bit Replicate operation + * the source is 'value' with a fixed size 'width' set of bits. + * value is replicated to fill out 8/16/32/64 bits as determined by 'size'. + * + * Example + * value is '11000011' (0xE3), width is 8 and size is EA_8BYTE + * result is '11000011 11000011 11000011 11000011 11000011 11000011 11000011 11000011' + * 0xE3E3E3E3E3E3E3E3 + */ - if (validVar1 && validVar2) - { - id = emitNewInstrLclVarPair(attr, imm); - id->idAddr()->iiaLclVar.initLclVarAddr(varx1, offs1); - id->idSetIsLclVar(); +/*static*/ UINT64 emitter::Replicate_helper(UINT64 value, unsigned width, emitAttr size) +{ + unsigned immWidth = getBitWidth(size); + assert(width <= immWidth); - emitGetLclVarPairLclVar2(id)->initLclVarAddr(varx2, offs2); - } - else + UINT64 result = value; + unsigned filledBits = width; + + while (filledBits < immWidth) { - id = emitNewInstrCns(attr, imm); - if (validVar1) - { - id->idAddr()->iiaLclVar.initLclVarAddr(varx1, offs1); - id->idSetIsLclVar(); - } - if (validVar2) - { - id->idAddr()->iiaLclVar.initLclVarAddr(varx2, offs2); - id->idSetIsLclVar(); - } + value <<= width; + result |= value; + filledBits += width; } + return result; +} - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); +/************************************************************************ + * + * Convert an imm(N,r,s) into a 64-bit immediate + * inputs 'bmImm' a bitMaskImm struct + * 'size' specifies the size of the result (8/16/32/64 bits) + */ - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); +/*static*/ INT64 emitter::emitDecodeBitMaskImm(const emitter::bitMaskImm bmImm, emitAttr size) +{ + unsigned N = bmImm.immN; // read the N,R and S values from the 'bitMaskImm' encoding + unsigned R = bmImm.immR; + unsigned S = bmImm.immS; - // Record the attribute for the second register in the pair - if (EA_IS_GCREF(attr2)) - { - id->idGCrefReg2(GCT_GCREF); - } - else if (EA_IS_BYREF(attr2)) + unsigned elemWidth = 64; // used when N == 1 + + if (N == 0) // find the smaller elemWidth when N == 0 { - id->idGCrefReg2(GCT_BYREF); + // Scan S for the highest bit not set + elemWidth = 32; + for (unsigned bitNum = 5; bitNum > 0; bitNum--) + { + unsigned oneBit = elemWidth; + if ((S & oneBit) == 0) + break; + elemWidth /= 2; + } } else { - id->idGCrefReg2(GCT_NONE); + assert(size == EA_8BYTE); } -#ifdef DEBUG - id->idDebugOnlyInfo()->idVarRefOffs = var1RefsOffs; - id->idDebugOnlyInfo()->idVarRefOffs2 = var2RefsOffs; -#endif - dispIns(id); - appendToCurIG(id); + unsigned maskSR = elemWidth - 1; + + S &= maskSR; + R &= maskSR; + + // encoding for S is one less than the number of consecutive one bits + S++; // Number of consecutive ones to generate in 'welem' + + // At this point: + // + // 'elemWidth' is the number of bits that we will use for the ROR and Replicate operations + // 'S' is the number of consecutive 1 bits for the immediate + // 'R' is the number of bits that we will Rotate Right the immediate + // 'size' selects the final size of the immediate that we return (64 or 32 bits) + + assert(S < elemWidth); // 'elemWidth' consecutive one's is a reserved encoding + + UINT64 welem; + UINT64 wmask; + + welem = (1ULL << S) - 1; + + wmask = ROR_helper(welem, R, elemWidth); + wmask = Replicate_helper(wmask, elemWidth, size); + + return wmask; } /***************************************************************************** * - * Add an instruction referencing three registers and a constant. + * Check if an immediate can use the left shifted by 12 bits encoding */ -void emitter::emitIns_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */, - emitAttr attrReg2 /* = EA_UNKNOWN */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +/*static*/ bool emitter::canEncodeWithShiftImmBy12(INT64 imm) { - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - bool isLdSt = false; - bool isSIMD = false; - bool isAddSub = false; - bool setFlags = false; - unsigned scale = 0; - - /* Figure out the encoding format of the instruction */ - switch (ins) + if (imm < 0) { - case INS_extr: - assert(insOptsNone(opt)); - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidImmShift(imm, size)); - fmt = IF_DR_3E; - break; - - case INS_and: - case INS_ands: - case INS_eor: - case INS_orr: - case INS_bic: - case INS_bics: - case INS_eon: - case INS_orn: - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidImmShift(imm, size)); - if (imm == 0) - { - assert(insOptsNone(opt)); // a zero imm, means no shift kind - fmt = IF_DR_3A; - } - else - { - assert(insOptsAnyShift(opt)); // a non-zero imm, must select shift kind - fmt = IF_DR_3B; - } - break; + imm = -imm; // convert to unsigned + } - case INS_fmul: // by element, imm[0..3] selects the element of reg3 - case INS_fmla: - case INS_fmls: - case INS_fmulx: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorElemsizeFloat(elemsize)); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - assert(opt != INS_OPTS_1D); // Reserved encoding - fmt = IF_DV_3BI; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert(isValidScalarDatasize(size)); - elemsize = size; - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - fmt = IF_DV_3DI; - } - break; + if (imm < 0) + { + return false; // Must be MIN_INT64 + } - case INS_mul: // by element, imm[0..7] selects the element of reg3 - case INS_mla: - case INS_mls: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - // Vector operation - assert(insOptsAnyArrangement(opt)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - elemsize = optGetElemsize(opt); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - // Only has encodings for H or S elemsize - assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)); - // Only has encodings for V0..V15 - if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) - { - noway_assert(!"Invalid reg3"); - } - fmt = IF_DV_3AI; - break; + if ((imm & 0xfff) != 0) // Now the low 12 bits all have to be zero + { + return false; + } - case INS_add: - case INS_sub: - setFlags = false; - isAddSub = true; - break; + imm >>= 12; // shift right by 12 bits - case INS_adds: - case INS_subs: - setFlags = true; - isAddSub = true; - break; + return (imm <= 0x0fff); // Does it fit in 12 bits +} - case INS_ldpsw: - scale = 2; - isLdSt = true; - break; +/***************************************************************************** + * + * Normalize the 'imm' so that the upper bits, as defined by 'size' are zero + */ - case INS_ldnp: - case INS_stnp: - assert(insOptsNone(opt)); // Can't use Pre/Post index on these two instructions - FALLTHROUGH; +/*static*/ INT64 emitter::normalizeImm64(INT64 imm, emitAttr size) +{ + unsigned immWidth = getBitWidth(size); + INT64 result = imm; - case INS_ldp: - case INS_stp: - // Is the target a vector register? - if (isVectorRegister(reg1)) - { - scale = NaturalScale_helper(size); - isSIMD = true; - } - else - { - scale = (size == EA_8BYTE) ? 3 : 2; - } - isLdSt = true; - fmt = IF_LS_3C; - break; + if (immWidth < 64) + { + // Check that 'imm' fits in 'immWidth' bits. Don't consider "sign" bits above width. + INT64 maxVal = 1LL << immWidth; + INT64 lowBitsMask = maxVal - 1; + INT64 hiBitsMask = ~lowBitsMask; + INT64 signBitsMask = + hiBitsMask | (1LL << (immWidth - 1)); // The high bits must be set, and the top bit (sign bit) must be set. + assert((imm < maxVal) || ((imm & signBitsMask) == signBitsMask)); - case INS_ld1: - case INS_ld2: - case INS_ld3: - case INS_ld4: - case INS_st1: - case INS_st2: - case INS_st3: - case INS_st4: - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrSP(reg2)); - assert(isGeneralRegister(reg3)); + // mask off the hiBits + result &= lowBitsMask; + } + return result; +} - assert(insOptsPostIndex(opt)); +/************************************************************************ + * + * returns true if 'imm' of 'size bits (8/16/32/64) can be encoded + * using the ARM64 'bitmask immediate' form. + * When a non-null value is passed for 'wbBMI' then this method + * writes back the 'N','S' and 'R' values use to encode this immediate + * + */ - elemsize = size; - assert(isValidVectorElemsize(elemsize)); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); +/*static*/ bool emitter::canEncodeBitMaskImm(INT64 imm, emitAttr size, emitter::bitMaskImm* wbBMI) +{ + unsigned immWidth = getBitWidth(size); + unsigned maxLen; - // Load/Store single structure post-indexed by a register - reg2 = encodingSPtoZR(reg2); - fmt = IF_LS_3G; + switch (size) + { + case EA_1BYTE: + maxLen = 3; break; - case INS_ext: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isValidVectorDatasize(size)); - assert(isValidArrangement(size, opt)); - assert((opt == INS_OPTS_8B) || (opt == INS_OPTS_16B)); - assert(isValidVectorIndex(size, EA_1BYTE, imm)); - fmt = IF_DV_3G; + case EA_2BYTE: + maxLen = 4; break; - case INS_smlal: - case INS_smlsl: - case INS_smull: - case INS_umlal: - case INS_umlsl: - case INS_umull: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_8BYTE); - assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); - elemsize = optGetElemsize(opt); - // Restricted to V0-V15 when element size is H. - if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) - { - assert(!"Invalid reg3"); - } - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - fmt = IF_DV_3AI; + case EA_4BYTE: + maxLen = 5; break; - case INS_sqdmlal: - case INS_sqdmlsl: - case INS_sqdmull: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(size == EA_8BYTE); - assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); - elemsize = optGetElemsize(opt); - fmt = IF_DV_3AI; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert((size == EA_2BYTE) || (size == EA_4BYTE)); - elemsize = size; - fmt = IF_DV_3EI; - } - // Restricted to V0-V15 when element size is H. - if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) - { - assert(!"Invalid reg3"); - } - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + case EA_8BYTE: + maxLen = 6; break; - case INS_sqdmulh: - case INS_sqrdmlah: - case INS_sqrdmlsh: - case INS_sqrdmulh: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - if (insOptsAnyArrangement(opt)) - { - // Vector operation - assert(isValidVectorDatasize(size)); - elemsize = optGetElemsize(opt); - assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)); - fmt = IF_DV_3AI; - } - else - { - // Scalar operation - assert(insOptsNone(opt)); - assert((size == EA_2BYTE) || (size == EA_4BYTE)); - elemsize = size; - fmt = IF_DV_3EI; - } - // Restricted to V0-V15 when element size is H. - if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) - { - assert(!"Invalid reg3"); - } - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + default: + assert(!"Invalid size"); + maxLen = 0; break; + } - case INS_smlal2: - case INS_smlsl2: - case INS_smull2: - case INS_sqdmlal2: - case INS_sqdmlsl2: - case INS_sqdmull2: - case INS_umlal2: - case INS_umlsl2: - case INS_umull2: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(size == EA_16BYTE); - assert((opt == INS_OPTS_8H) || (opt == INS_OPTS_4S)); - elemsize = optGetElemsize(opt); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - // Restricted to V0-V15 when element size is H - if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) + imm = normalizeImm64(imm, size); + + // Starting with len=1, elemWidth is 2 bits + // len=2, elemWidth is 4 bits + // len=3, elemWidth is 8 bits + // len=4, elemWidth is 16 bits + // len=5, elemWidth is 32 bits + // len=6, elemWidth is 64 bits + // + for (unsigned len = 1; (len <= maxLen); len++) + { + unsigned elemWidth = 1 << len; + UINT64 elemMask = ((UINT64)-1) >> (64 - elemWidth); + UINT64 tempImm = (UINT64)imm; // A working copy of 'imm' that we can mutate + UINT64 elemVal = tempImm & elemMask; // The low 'elemWidth' bits of 'imm' + + // Check for all 1's or 0's as these can't be encoded + if ((elemVal == 0) || (elemVal == elemMask)) + continue; + + // 'checkedBits' is the count of bits that are known to match 'elemVal' when replicated + unsigned checkedBits = elemWidth; // by definition the first 'elemWidth' bits match + + // Now check to see if each of the next bits match... + // + while (checkedBits < immWidth) + { + tempImm >>= elemWidth; + + UINT64 nextElem = tempImm & elemMask; + if (nextElem != elemVal) { - assert(!"Invalid reg3"); + // Not matching, exit this loop and checkedBits will not be equal to immWidth + break; } - fmt = IF_DV_3AI; - break; - case INS_sdot: - case INS_udot: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(((size == EA_8BYTE) && (opt == INS_OPTS_2S)) || ((size == EA_16BYTE) && (opt == INS_OPTS_4S))); - assert(isValidVectorIndex(EA_16BYTE, EA_4BYTE, imm)); - fmt = IF_DV_3AI; - break; + // The 'nextElem' is matching, so increment 'checkedBits' + checkedBits += elemWidth; + } - default: - // fallback to emit SVE instructions. - return emitInsSve_R_R_R_I(ins, attr, reg1, reg2, reg3, imm, opt, sopt); + // Did the full immediate contain bits that can be formed by repeating 'elemVal'? + if (checkedBits == immWidth) + { + // We are not quite done, since the only values that we can encode as a + // 'bitmask immediate' are those that can be formed by starting with a + // bit string of 0*1* that is rotated by some number of bits. + // + // We check to see if 'elemVal' can be formed using these restrictions. + // + // Observation: + // Rotating by one bit any value that passes these restrictions + // can be xor-ed with the original value and will result it a string + // of bits that have exactly two 1 bits: 'elemRorXor' + // Further the distance between the two one bits tells us the value + // of S and the location of the 1 bits tells us the value of R + // + // Some examples: (immWidth is 8) + // + // S=4,R=0 S=5,R=3 S=3,R=6 + // elemVal: 00001111 11100011 00011100 + // elemRor: 10000111 11110001 00001110 + // elemRorXor: 10001000 00010010 00010010 + // compute S 45678--- ---5678- ---3210- + // compute R 01234567 ---34567 ------67 - } // end switch (ins) + UINT64 elemRor = ROR_helper(elemVal, 1, elemWidth); // Rotate 'elemVal' Right by one bit + UINT64 elemRorXor = elemVal ^ elemRor; // Xor elemVal and elemRor - assert(insScalableOptsNone(sopt)); + // If we only have a two-bit change in elemROR then we can form a mask for this value + unsigned bitCount = 0; + UINT64 oneBit = 0x1; + unsigned R = elemWidth; // R is shift count for ROR (rotate right shift) + unsigned S = 0; // S is number of consecutive one bits + int incr = -1; - if (isLdSt) - { - assert(!isAddSub); - assert(isGeneralRegisterOrSP(reg3)); - assert(insOptsNone(opt) || insOptsIndexed(opt)); + // Loop over the 'elemWidth' bits in 'elemRorXor' + // + for (unsigned bitNum = 0; bitNum < elemWidth; bitNum++) + { + if (incr == -1) + { + R--; // We decrement R by one whenever incr is -1 + } + if (bitCount == 1) + { + S += incr; // We incr/decr S, after we find the first one bit in 'elemRorXor' + } + + // Is this bit position a 1 bit in 'elemRorXor'? + // + if (oneBit & elemRorXor) + { + bitCount++; + // Is this the first 1 bit that we found in 'elemRorXor'? + if (bitCount == 1) + { + // Does this 1 bit represent a transition to zero bits? + bool toZeros = ((oneBit & elemVal) != 0); + if (toZeros) + { + // S :: Count down from elemWidth + S = elemWidth; + incr = -1; + } + else // this 1 bit represent a transition to one bits. + { + // S :: Count up from zero + S = 0; + incr = +1; + } + } + else // bitCount > 1 + { + // We found the second (or third...) 1 bit in 'elemRorXor' + incr = 0; // stop decrementing 'R' - if (isSIMD) - { - assert(isValidVectorLSPDatasize(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert((scale >= 2) && (scale <= 4)); - } - else - { - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegisterOrZR(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert((scale == 2) || (scale == 3)); - } + if (bitCount > 2) + { + // More than 2 transitions from 0/1 in 'elemVal' + // This means that 'elemVal' can't be encoded + // using a 'bitmask immediate'. + // + // Furthermore, it will continue to fail + // with any larger 'len' that we try. + // so just return false. + // + return false; + } + } + } - // Load/Store Pair reserved encodings: - if (emitInsIsLoad(ins)) - { - assert(reg1 != reg2); - } - if (insOptsIndexed(opt)) - { - assert(reg1 != reg3); - assert(reg2 != reg3); - } + // shift oneBit left by one bit to test the next position + oneBit <<= 1; + } - reg3 = encodingSPtoZR(reg3); + // We expect that bitCount will always be two at this point + // but just in case return false for any bad cases. + // + assert(bitCount == 2); + if (bitCount != 2) + return false; - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - if (imm == 0) - { - assert(insOptsNone(opt)); // PRE/POST Index doesn't make sense with an immediate of zero + // Perform some sanity checks on the values of 'S' and 'R' + assert(S > 0); + assert(S < elemWidth); + assert(R < elemWidth); - fmt = IF_LS_3B; - } - else - { - if ((imm & mask) == 0) + // Does the caller want us to return the N,R,S encoding values? + // + if (wbBMI != nullptr) { - imm >>= scale; // The immediate is scaled by the size of the ld/st - if ((imm >= -64) && (imm <= 63)) + // The encoding used for S is one less than the + // number of consecutive one bits + S--; + + if (len == 6) { - fmt = IF_LS_3C; + wbBMI->immN = 1; } + else + { + wbBMI->immN = 0; + // The encoding used for 'S' here is a bit peculiar. + // + // The upper bits need to be complemented, followed by a zero bit + // then the value of 'S-1' + // + unsigned upperBitsOfS = 64 - (1 << (len + 1)); + S |= upperBitsOfS; + } + wbBMI->immR = R; + wbBMI->immS = S; + + // Verify that what we are returning is correct. + assert(imm == emitDecodeBitMaskImm(*wbBMI, size)); } -#ifdef DEBUG - if (fmt != IF_LS_3C) - { - assert(!"Instruction cannot be encoded: IF_LS_3C"); - } -#endif + // Tell the caller that we can successfully encode this immediate + // using a 'bitmask immediate'. + // + return true; } } - else if (isAddSub) - { - bool reg2IsSP = (reg2 == REG_SP); - assert(!isLdSt); - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg3)); + return false; +} - if (setFlags || insOptsAluShift(opt)) // Can't encode SP in reg1 with setFlags or AluShift option - { - assert(isGeneralRegisterOrZR(reg1)); - } - else - { - assert(isGeneralRegisterOrSP(reg1)); - reg1 = encodingSPtoZR(reg1); - } +/************************************************************************ + * + * Convert an imm(i16,hw) into a 32/64-bit immediate + * inputs 'hwImm' a halfwordImm struct + * 'size' specifies the size of the result (64 or 32 bits) + */ - if (insOptsAluShift(opt)) // Can't encode SP in reg2 with AluShift option - { - assert(isGeneralRegister(reg2)); - } - else - { - assert(isGeneralRegisterOrSP(reg2)); - reg2 = encodingSPtoZR(reg2); - } +/*static*/ INT64 emitter::emitDecodeHalfwordImm(const emitter::halfwordImm hwImm, emitAttr size) +{ + assert(isValidGeneralDatasize(size)); // Only EA_4BYTE or EA_8BYTE forms - if (insOptsAnyExtend(opt)) - { - assert((imm >= 0) && (imm <= 4)); + unsigned hw = hwImm.immHW; + INT64 val = (INT64)hwImm.immVal; - fmt = IF_DR_3C; - } - else if (insOptsAluShift(opt)) - { - // imm should be non-zero and in [1..63] - assert(isValidImmShift(imm, size) && (imm != 0)); - fmt = IF_DR_3B; - } - else if (imm == 0) - { - assert(insOptsNone(opt)); + assert((hw <= 1) || (size == EA_8BYTE)); - if (reg2IsSP) - { - // To encode the SP register as reg2 we must use the IF_DR_3C encoding - // and also specify a LSL of zero (imm == 0) - opt = INS_OPTS_LSL; - fmt = IF_DR_3C; - } - else - { - fmt = IF_DR_3A; - } - } - else - { - assert(!"Instruction cannot be encoded: Add/Sub IF_DR_3A"); - } - } + INT64 result = val << (16 * hw); + return result; +} - assert(fmt != IF_NONE); +/************************************************************************ + * + * returns true if 'imm' of 'size' bits (32/64) can be encoded + * using the ARM64 'halfword immediate' form. + * When a non-null value is passed for 'wbHWI' then this method + * writes back the 'immHW' and 'immVal' values use to encode this immediate + * + */ - instrDesc* id = emitNewInstrCns(attr, imm); +/*static*/ bool emitter::canEncodeHalfwordImm(INT64 imm, emitAttr size, emitter::halfwordImm* wbHWI) +{ + assert(isValidGeneralDatasize(size)); // Only EA_4BYTE or EA_8BYTE forms - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); + unsigned immWidth = (size == EA_8BYTE) ? 64 : 32; + unsigned maxHW = (size == EA_8BYTE) ? 4 : 2; - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); + // setup immMask to a (EA_4BYTE) 0x00000000_FFFFFFFF or (EA_8BYTE) 0xFFFFFFFF_FFFFFFFF + const UINT64 immMask = ((UINT64)-1) >> (64 - immWidth); + const INT64 mask16 = (INT64)0xFFFF; - // Record the attribute for the second register in the pair - id->idGCrefReg2(GCT_NONE); - if (attrReg2 != EA_UNKNOWN) - { - // Record the attribute for the second register in the pair - assert((fmt == IF_LS_3B) || (fmt == IF_LS_3C)); - if (EA_IS_GCREF(attrReg2)) - { - id->idGCrefReg2(GCT_GCREF); - } - else if (EA_IS_BYREF(attrReg2)) - { - id->idGCrefReg2(GCT_BYREF); - } - } + imm = normalizeImm64(imm, size); - dispIns(id); - appendToCurIG(id); -} + // Try each of the valid hw shift sizes + for (unsigned hw = 0; (hw < maxHW); hw++) + { + INT64 curMask = mask16 << (hw * 16); // Represents the mask of the bits in the current halfword + INT64 checkBits = immMask & ~curMask; -/***************************************************************************** - * - * Add an instruction referencing three registers and two constants. - */ + // Excluding the current halfword (using ~curMask) + // does the immediate have zero bits in every other bit that we care about? + // note we care about all 64-bits for EA_8BYTE + // and we care about the lowest 32 bits for EA_4BYTE + // + if ((imm & checkBits) == 0) + { + // Does the caller want us to return the imm(i16,hw) encoding values? + // + if (wbHWI != nullptr) + { + INT64 val = ((imm & curMask) >> (hw * 16)) & mask16; + wbHWI->immHW = hw; + wbHWI->immVal = val; -void emitter::emitIns_R_R_R_I_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm1, - ssize_t imm2, - insOpts opt) -{ - // Currently, only SVE instructions use this format. - emitInsSve_R_R_R_I_I(ins, attr, reg1, reg2, reg3, imm1, imm2, opt); + // Verify that what we are returning is correct. + assert(imm == emitDecodeHalfwordImm(*wbHWI, size)); + } + // Tell the caller that we can successfully encode this immediate + // using a 'halfword immediate'. + // + return true; + } + } + return false; } -/***************************************************************************** +/************************************************************************ * - * Add an instruction referencing three registers, with an extend option + * Convert an imm(i8,sh) into a 16/32-bit immediate + * inputs 'bsImm' a byteShiftedImm struct + * 'size' specifies the size of the result (16 or 32 bits) */ -void emitter::emitIns_R_R_R_Ext(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - insOpts opt, /* = INS_OPTS_NONE */ - int shiftAmount) /* = -1 -- unset */ +/*static*/ UINT32 emitter::emitDecodeByteShiftedImm(const emitter::byteShiftedImm bsImm, emitAttr size) { - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - bool isSIMD = false; - int scale = -1; + bool onesShift = (bsImm.immOnes == 1); + unsigned bySh = bsImm.immBY; // Num Bytes to shift 0,1,2,3 + UINT32 result = (UINT32)bsImm.immVal; // 8-bit immediate - /* Figure out the encoding format of the instruction */ - switch (ins) + if (bySh > 0) { - case INS_ldrb: - case INS_ldrsb: - case INS_strb: - scale = 0; - break; - - case INS_ldrh: - case INS_ldrsh: - case INS_strh: - scale = 1; - break; - - case INS_ldrsw: - scale = 2; - break; + assert((size == EA_2BYTE) || (size == EA_4BYTE)); // Only EA_2BYTE or EA_4BYTE forms + if (size == EA_2BYTE) + { + assert(bySh < 2); + } + else + { + assert(bySh < 4); + } - case INS_ldr: - case INS_str: - // Is the target a vector register? - if (isVectorRegister(reg1)) - { - assert(isValidVectorLSDatasize(size)); - scale = NaturalScale_helper(size); - isSIMD = true; - } - else - { - assert(isValidGeneralDatasize(size)); - scale = (size == EA_8BYTE) ? 3 : 2; - } + result <<= (8 * bySh); - break; + if (onesShift) + { + result |= ((1 << (8 * bySh)) - 1); + } + } + return result; +} - default: - unreached(); - break; +/************************************************************************ + * + * returns true if 'imm' of 'size' bits (16/32) can be encoded + * using the ARM64 'byteShifted immediate' form. + * When a non-null value is passed for 'wbBSI' then this method + * writes back the 'immBY' and 'immVal' values use to encode this immediate + * + */ - } // end switch (ins) +/*static*/ bool emitter::canEncodeByteShiftedImm(INT64 imm, + emitAttr size, + bool allow_MSL, + emitter::byteShiftedImm* wbBSI) +{ + bool canEncode = false; + bool onesShift = false; // true if we use the shifting ones variant + unsigned bySh = 0; // number of bytes to shift: 0, 1, 2, 3 + unsigned imm8 = 0; // immediate to use in the encoding - assert(scale != -1); - assert(insOptsLSExtend(opt)); + imm = normalizeImm64(imm, size); - if (isSIMD) + if (size == EA_1BYTE) { - assert(isValidVectorLSDatasize(size)); - assert(isVectorRegister(reg1)); + imm8 = (unsigned)imm; + assert(imm8 < 0x100); + canEncode = true; } - else + else if (size == EA_8BYTE) { - assert(isValidGeneralLSDatasize(size)); - assert(isGeneralRegisterOrZR(reg1)); + imm8 = (unsigned)imm; + assert(imm8 < 0x100); + canEncode = true; } - - assert(isGeneralRegisterOrSP(reg2)); - assert(isGeneralRegister(reg3)); - - // Load/Store reserved encodings: - if (insOptsIndexed(opt)) + else { - assert(reg1 != reg2); - } + assert((size == EA_2BYTE) || (size == EA_4BYTE)); // Only EA_2BYTE or EA_4BYTE forms - if (shiftAmount == -1) - { - shiftAmount = insOptsLSL(opt) ? scale : 0; - } + unsigned immWidth = (size == EA_4BYTE) ? 32 : 16; + unsigned maxBY = (size == EA_4BYTE) ? 4 : 2; - assert((shiftAmount == scale) || (shiftAmount == 0)); + // setup immMask to a (EA_2BYTE) 0x0000FFFF or (EA_4BYTE) 0xFFFFFFFF + const UINT32 immMask = ((UINT32)-1) >> (32 - immWidth); + const INT32 mask8 = (INT32)0xFF; - reg2 = encodingSPtoZR(reg2); - fmt = IF_LS_3A; + // Try each of the valid by shift sizes + for (bySh = 0; (bySh < maxBY); bySh++) + { + INT32 curMask = mask8 << (bySh * 8); // Represents the mask of the bits in the current byteShifted + INT32 checkBits = immMask & ~curMask; + INT32 immCheck = (imm & checkBits); - instrDesc* id = emitNewInstr(attr); + // Excluding the current byte (using ~curMask) + // does the immediate have zero bits in every other bit that we care about? + // or can be use the shifted one variant? + // note we care about all 32-bits for EA_4BYTE + // and we care about the lowest 16 bits for EA_2BYTE + // + if (immCheck == 0) + { + canEncode = true; + } - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); + // MSL is only supported for 32-bit. + if (allow_MSL && (size == EA_4BYTE)) + { + if ((bySh == 1) && (immCheck == 0xFF)) + { + canEncode = true; + onesShift = true; + } + else if ((bySh == 2) && (immCheck == 0xFFFF)) + { + canEncode = true; + onesShift = true; + } + } + if (canEncode) + { + imm8 = (unsigned)(((imm & curMask) >> (bySh * 8)) & mask8); + break; + } + } + } - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idReg3Scaled(shiftAmount == scale); + if (canEncode) + { + // Does the caller want us to return the imm(i8,bySh) encoding values? + // + if (wbBSI != nullptr) + { + wbBSI->immOnes = onesShift; + wbBSI->immBY = bySh; + wbBSI->immVal = imm8; - dispIns(id); - appendToCurIG(id); + // Verify that what we are returning is correct. + assert(imm == emitDecodeByteShiftedImm(*wbBSI, size)); + } + // Tell the caller that we can successfully encode this immediate + // using a 'byteShifted immediate'. + // + return true; + } + return false; } -/***************************************************************************** +/************************************************************************ * - * Add an instruction referencing two registers and two constants. + * Convert a 'float 8-bit immediate' into a double. + * inputs 'fpImm' a floatImm8 struct */ -void emitter::emitIns_R_R_I_I( - instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, int imm1, int imm2, insOpts opt) +/*static*/ double emitter::emitDecodeFloatImm8(const emitter::floatImm8 fpImm) { - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - size_t immOut = 0; // composed from imm1 and imm2 and stored in the instrDesc - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - int lsb; - int width; - bitMaskImm bmi; - unsigned registerListSize; - - case INS_bfm: - case INS_sbfm: - case INS_ubfm: - assert(isGeneralRegister(reg1)); - assert((ins == INS_bfm) ? isGeneralRegisterOrZR(reg2) : isGeneralRegister(reg2)); - assert(isValidImmShift(imm1, size)); - assert(isValidImmShift(imm2, size)); - assert(insOptsNone(opt)); - bmi.immNRS = 0; - bmi.immN = (size == EA_8BYTE); - bmi.immR = imm1; - bmi.immS = imm2; - immOut = bmi.immNRS; - fmt = IF_DI_2D; - break; - - case INS_bfi: - case INS_sbfiz: - case INS_ubfiz: - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - lsb = getBitWidth(size) - imm1; - width = imm2 - 1; - assert(isValidImmShift(lsb, size)); - assert(isValidImmShift(width, size)); - assert(insOptsNone(opt)); - bmi.immNRS = 0; - bmi.immN = (size == EA_8BYTE); - bmi.immR = lsb; - bmi.immS = width; - immOut = bmi.immNRS; - fmt = IF_DI_2D; - break; + unsigned sign = fpImm.immSign; + unsigned exp = fpImm.immExp ^ 0x4; + unsigned mant = fpImm.immMant + 16; + unsigned scale = 16 * 8; - case INS_bfxil: - case INS_sbfx: - case INS_ubfx: - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - lsb = imm1; - width = imm2 + imm1 - 1; - assert(isValidImmShift(lsb, size)); - assert(isValidImmShift(width, size)); - assert(insOptsNone(opt)); - bmi.immNRS = 0; - bmi.immN = (size == EA_8BYTE); - bmi.immR = imm1; - bmi.immS = imm2 + imm1 - 1; - immOut = bmi.immNRS; - fmt = IF_DI_2D; - break; + while (exp > 0) + { + scale /= 2; + exp--; + } - case INS_mov: - case INS_ins: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - elemsize = size; - assert(isValidVectorElemsize(elemsize)); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm1)); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm2)); - assert(insOptsNone(opt)); - immOut = (imm1 << 4) + imm2; - fmt = IF_DV_2F; - break; + double result = ((double)mant) / ((double)scale); + if (sign == 1) + { + result = -result; + } - case INS_ld1: - case INS_ld2: - case INS_ld3: - case INS_ld4: - case INS_st1: - case INS_st2: - case INS_st3: - case INS_st4: - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrSP(reg2)); + return result; +} - elemsize = size; - assert(isValidVectorElemsize(elemsize)); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm1)); +/************************************************************************ + * + * returns true if the 'immDbl' can be encoded using the 'float 8-bit immediate' form. + * also returns the encoding if wbFPI is non-null + * + */ - registerListSize = insGetRegisterListSize(ins); - assert((elemsize * registerListSize) == (unsigned)imm2); - assert(insOptsPostIndex(opt)); +/*static*/ bool emitter::canEncodeFloatImm8(double immDbl, emitter::floatImm8* wbFPI) +{ + bool canEncode = false; + double val = immDbl; - // Load/Store single structure post-indexed by an immediate - reg2 = encodingSPtoZR(reg2); - immOut = imm1; - fmt = IF_LS_2G; - break; + int sign = 0; + if (val < 0.0) + { + val = -val; + sign = 1; + } - default: - unreached(); - break; + int exp = 0; + while ((val < 1.0) && (exp >= -4)) + { + val *= 2.0; + exp--; } - assert(fmt != IF_NONE); + while ((val >= 2.0) && (exp <= 5)) + { + val *= 0.5; + exp++; + } + exp += 3; + val *= 16.0; + int ival = (int)val; - instrDesc* id = emitNewInstrSC(attr, immOut); + if ((exp >= 0) && (exp <= 7)) + { + if (val == (double)ival) + { + canEncode = true; - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); + if (wbFPI != nullptr) + { + ival -= 16; + assert((ival >= 0) && (ival <= 15)); - id->idReg1(reg1); - id->idReg2(reg2); + wbFPI->immSign = sign; + wbFPI->immExp = exp ^ 0x4; + wbFPI->immMant = ival; + unsigned imm8 = wbFPI->immFPIVal; + assert((imm8 >= 0) && (imm8 <= 0xff)); + } + } + } - dispIns(id); - appendToCurIG(id); + return canEncode; } /***************************************************************************** * - * Add an instruction referencing four registers. + * For the given 'ins' returns the reverse instruction + * if one exists, otherwise returns INS_INVALID */ -void emitter::emitIns_R_R_R_R(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - regNumber reg4, - insOpts opt /* = INS_OPTS_NONE*/, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +/*static*/ instruction emitter::insReverse(instruction ins) { - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_madd: - case INS_msub: - case INS_smaddl: - case INS_smsubl: - case INS_umaddl: - case INS_umsubl: - assert(isValidGeneralDatasize(size)); - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isGeneralRegister(reg4)); - assert(insScalableOptsNone(sopt)); - fmt = IF_DR_4A; - break; + case INS_add: + return INS_sub; + case INS_adds: + return INS_subs; - case INS_fmadd: - case INS_fmsub: - case INS_fnmadd: - case INS_fnmsub: - // Scalar operation - assert(isValidScalarDatasize(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isVectorRegister(reg4)); - assert(insScalableOptsNone(sopt)); - fmt = IF_DV_4A; - break; + case INS_sub: + return INS_add; + case INS_subs: + return INS_adds; - case INS_invalid: - fmt = IF_NONE; - break; + case INS_cmp: + return INS_cmn; + case INS_cmn: + return INS_cmp; + + case INS_ccmp: + return INS_ccmn; + case INS_ccmn: + return INS_ccmp; - // Fallback handles emitting the SVE instructions. default: - return emitInsSve_R_R_R_R(ins, attr, reg1, reg2, reg3, reg4, opt, sopt); + return INS_invalid; } - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstr(attr); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idReg4(reg4); - - dispIns(id); - appendToCurIG(id); } /***************************************************************************** * - * Add an instruction referencing four registers and a constant. + * For the given 'datasize' and 'elemsize', make the proper arrangement option + * returns the insOpts that specifies the vector register arrangement + * if one does not exist returns INS_OPTS_NONE */ -void emitter::emitIns_R_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - regNumber reg4, - ssize_t imm, - insOpts opt /* = INS_OPT_NONE*/) +/*static*/ insOpts emitter::optMakeArrangement(emitAttr datasize, emitAttr elemsize) { - // Currently, only SVE instructions use this format. - emitInsSve_R_R_R_R_I(ins, attr, reg1, reg2, reg3, reg4, imm, opt); + insOpts result = INS_OPTS_NONE; + + if (datasize == EA_8BYTE) + { + switch (elemsize) + { + case EA_1BYTE: + result = INS_OPTS_8B; + break; + case EA_2BYTE: + result = INS_OPTS_4H; + break; + case EA_4BYTE: + result = INS_OPTS_2S; + break; + case EA_8BYTE: + result = INS_OPTS_1D; + break; + default: + unreached(); + break; + } + } + else if (datasize == EA_16BYTE) + { + switch (elemsize) + { + case EA_1BYTE: + result = INS_OPTS_16B; + break; + case EA_2BYTE: + result = INS_OPTS_8H; + break; + case EA_4BYTE: + result = INS_OPTS_4S; + break; + case EA_8BYTE: + result = INS_OPTS_2D; + break; + default: + unreached(); + break; + } + } + return result; } /***************************************************************************** * - * Add an instruction referencing a register and a condition code + * For the given 'datasize' and arrangement 'opts' + * returns true is the pair specifies a valid arrangement */ +/*static*/ bool emitter::isValidArrangement(emitAttr datasize, insOpts opt) +{ + if (datasize == EA_8BYTE) + { + if ((opt == INS_OPTS_8B) || (opt == INS_OPTS_4H) || (opt == INS_OPTS_2S) || (opt == INS_OPTS_1D)) + { + return true; + } + } + else if (datasize == EA_16BYTE) + { + if ((opt == INS_OPTS_16B) || (opt == INS_OPTS_8H) || (opt == INS_OPTS_4S) || (opt == INS_OPTS_2D)) + { + return true; + } + } + return false; +} -void emitter::emitIns_R_COND(instruction ins, emitAttr attr, regNumber reg, insCond cond) +//------------------------------------------------------------------------ +// insGetRegisterListSize: Returns a size of the register list a given instruction operates on. +// +// Arguments: +// ins - An instruction which uses a register list +// (e.g. ld1 (2 registers), ld1r, st1, tbl, tbx). +// +// Return value: +// A number of consecutive SIMD and floating-point registers the instruction loads to/store from. +// +/*static*/ unsigned emitter::insGetRegisterListSize(instruction ins) { - insFormat fmt = IF_NONE; - condFlagsImm cfi; - cfi.immCFVal = 0; + unsigned registerListSize = 0; - /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_cset: - case INS_csetm: - assert(isGeneralRegister(reg)); - cfi.cond = cond; - fmt = IF_DR_1D; + case INS_ld1: + case INS_ld1r: + case INS_st1: + case INS_tbl: + case INS_tbx: + registerListSize = 1; break; - default: - unreached(); + case INS_ld1_2regs: + case INS_ld2: + case INS_ld2r: + case INS_st1_2regs: + case INS_st2: + case INS_tbl_2regs: + case INS_tbx_2regs: + registerListSize = 2; break; - } // end switch (ins) - - assert(fmt != IF_NONE); - assert(isValidImmCond(cfi.immCFVal)); - - instrDesc* id = emitNewInstrSC(attr, cfi.immCFVal); + case INS_ld1_3regs: + case INS_ld3: + case INS_ld3r: + case INS_st1_3regs: + case INS_st3: + case INS_tbl_3regs: + case INS_tbx_3regs: + registerListSize = 3; + break; - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); + case INS_ld1_4regs: + case INS_ld4: + case INS_ld4r: + case INS_st1_4regs: + case INS_st4: + case INS_tbl_4regs: + case INS_tbx_4regs: + registerListSize = 4; + break; - id->idReg1(reg); + default: + assert(!"Unexpected instruction"); + break; + } - dispIns(id); - appendToCurIG(id); + return registerListSize; } -/***************************************************************************** - * - * Add an instruction referencing two registers and a condition code - */ - -void emitter::emitIns_R_R_COND(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, insCond cond) +// For the given 'arrangement' returns the 'datasize' specified by the vector register arrangement +// asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed +// +/*static*/ emitAttr emitter::optGetDatasize(insOpts arrangement) { - insFormat fmt = IF_NONE; - condFlagsImm cfi; - cfi.immCFVal = 0; - - /* Figure out the encoding format of the instruction */ - switch (ins) + if ((arrangement == INS_OPTS_8B) || (arrangement == INS_OPTS_4H) || (arrangement == INS_OPTS_2S) || + (arrangement == INS_OPTS_1D)) { - case INS_cinc: - case INS_cinv: - case INS_cneg: - assert(isGeneralRegister(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - cfi.cond = cond; - fmt = IF_DR_2D; - break; - default: - unreached(); - break; - - } // end switch (ins) - - assert(fmt != IF_NONE); - assert(isValidImmCond(cfi.immCFVal)); - - instrDesc* id = emitNewInstrSC(attr, cfi.immCFVal); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - - id->idReg1(reg1); - id->idReg2(reg2); - - dispIns(id); - appendToCurIG(id); + return EA_8BYTE; + } + else if ((arrangement == INS_OPTS_16B) || (arrangement == INS_OPTS_8H) || (arrangement == INS_OPTS_4S) || + (arrangement == INS_OPTS_2D)) + { + return EA_16BYTE; + } + else + { + assert(!" invalid 'arrangement' value"); + return EA_UNKNOWN; + } } -/***************************************************************************** - * - * Add an instruction referencing two registers and a condition code - */ - -void emitter::emitIns_R_R_R_COND( - instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, regNumber reg3, insCond cond) +// For the given 'arrangement' returns the 'elemsize' specified by the vector register arrangement +// asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed +// +/*static*/ emitAttr emitter::optGetElemsize(insOpts arrangement) { - insFormat fmt = IF_NONE; - condFlagsImm cfi; - cfi.immCFVal = 0; - - /* Figure out the encoding format of the instruction */ - switch (ins) + if ((arrangement == INS_OPTS_8B) || (arrangement == INS_OPTS_16B)) { - case INS_csel: - case INS_csinc: - case INS_csinv: - case INS_csneg: - assert(isGeneralRegister(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert(isGeneralRegisterOrZR(reg3)); - cfi.cond = cond; - fmt = IF_DR_3D; - break; - - default: - unreached(); - break; - - } // end switch (ins) - - assert(fmt != IF_NONE); - assert(isValidImmCond(cfi.immCFVal)); - - instrDesc* id = emitNewInstr(attr); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); + return EA_1BYTE; + } + else if ((arrangement == INS_OPTS_4H) || (arrangement == INS_OPTS_8H)) + { + return EA_2BYTE; + } + else if ((arrangement == INS_OPTS_2S) || (arrangement == INS_OPTS_4S)) + { + return EA_4BYTE; + } + else if ((arrangement == INS_OPTS_1D) || (arrangement == INS_OPTS_2D)) + { + return EA_8BYTE; + } + else + { + assert(!" invalid 'arrangement' value"); + return EA_UNKNOWN; + } +} - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idSmallCns(cfi.immCFVal); +/*static*/ insOpts emitter::optWidenElemsizeArrangement(insOpts arrangement) +{ + if ((arrangement == INS_OPTS_8B) || (arrangement == INS_OPTS_16B)) + { + return INS_OPTS_8H; + } + else if ((arrangement == INS_OPTS_4H) || (arrangement == INS_OPTS_8H)) + { + return INS_OPTS_4S; + } + else if ((arrangement == INS_OPTS_2S) || (arrangement == INS_OPTS_4S)) + { + return INS_OPTS_2D; + } + else + { + assert(!" invalid 'arrangement' value"); + return INS_OPTS_NONE; + } +} - dispIns(id); - appendToCurIG(id); +/*static*/ emitAttr emitter::widenDatasize(emitAttr datasize) +{ + if (datasize == EA_1BYTE) + { + return EA_2BYTE; + } + else if (datasize == EA_2BYTE) + { + return EA_4BYTE; + } + else if (datasize == EA_4BYTE) + { + return EA_8BYTE; + } + else + { + assert(!" invalid 'datasize' value"); + return EA_UNKNOWN; + } } -/***************************************************************************** - * - * Add an instruction referencing two registers the flags and a condition code - */ - -void emitter::emitIns_R_R_FLAGS_COND( - instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, insCflags flags, insCond cond) +// For the given 'srcArrangement' returns the "widen" 'dstArrangement' specifying the destination vector register +// arrangement +// asserts and returns INS_OPTS_NONE if an invalid 'srcArrangement' value is passed +// +/*static*/ insOpts emitter::optWidenDstArrangement(insOpts srcArrangement) { - insFormat fmt = IF_NONE; - condFlagsImm cfi; - cfi.immCFVal = 0; + insOpts dstArrangement = INS_OPTS_NONE; - /* Figure out the encoding format of the instruction */ - switch (ins) + switch (srcArrangement) { - case INS_ccmp: - case INS_ccmn: - assert(isGeneralRegister(reg1)); - assert(isGeneralRegister(reg2)); - cfi.flags = flags; - cfi.cond = cond; - fmt = IF_DR_2I; - break; - default: - unreached(); + case INS_OPTS_8B: + dstArrangement = INS_OPTS_4H; break; - } // end switch (ins) - - assert(fmt != IF_NONE); - assert(isValidImmCondFlags(cfi.immCFVal)); - - instrDesc* id = emitNewInstrSC(attr, cfi.immCFVal); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - id->idReg1(reg1); - id->idReg2(reg2); + case INS_OPTS_16B: + dstArrangement = INS_OPTS_8H; + break; - dispIns(id); - appendToCurIG(id); -} + case INS_OPTS_4H: + dstArrangement = INS_OPTS_2S; + break; -/***************************************************************************** - * - * Add an instruction referencing a register, an immediate, the flags and a condition code - */ + case INS_OPTS_8H: + dstArrangement = INS_OPTS_4S; + break; -void emitter::emitIns_R_I_FLAGS_COND( - instruction ins, emitAttr attr, regNumber reg, ssize_t imm, insCflags flags, insCond cond) -{ - insFormat fmt = IF_NONE; - condFlagsImm cfi; - cfi.immCFVal = 0; + case INS_OPTS_2S: + dstArrangement = INS_OPTS_1D; + break; - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_ccmp: - case INS_ccmn: - assert(isGeneralRegister(reg)); - if (imm < 0) - { - ins = insReverse(ins); - imm = -imm; - } - if (isValidUimm<5>(imm)) - { - cfi.imm5 = imm; - cfi.flags = flags; - cfi.cond = cond; - fmt = IF_DI_1F; - } - else - { - assert(!"Instruction cannot be encoded: ccmp/ccmn imm5"); - } + case INS_OPTS_4S: + dstArrangement = INS_OPTS_2D; break; + default: - unreached(); + assert(!" invalid 'srcArrangement' value"); break; - } // end switch (ins) - - assert(fmt != IF_NONE); - assert(isValidImmCondFlagsImm5(cfi.immCFVal)); - - instrDesc* id = emitNewInstrSC(attr, cfi.immCFVal); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - - id->idReg1(reg); + } - dispIns(id); - appendToCurIG(id); + return dstArrangement; } -/***************************************************************************** - * - * Add a memory barrier instruction with a 'barrier' immediate - */ - -void emitter::emitIns_BARR(instruction ins, insBarrier barrier) +// For the given 'conversion' returns the 'dstsize' specified by the conversion option +/*static*/ emitAttr emitter::optGetDstsize(insOpts conversion) { - insFormat fmt = IF_NONE; - ssize_t imm = 0; - - /* Figure out the encoding format of the instruction */ - switch (ins) + switch (conversion) { - case INS_dsb: - case INS_dmb: - case INS_isb: - - fmt = IF_SI_0B; - imm = (ssize_t)barrier; - break; - default: - unreached(); - break; - } // end switch (ins) - - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSC(EA_8BYTE, imm); + case INS_OPTS_S_TO_8BYTE: + case INS_OPTS_D_TO_8BYTE: + case INS_OPTS_4BYTE_TO_D: + case INS_OPTS_8BYTE_TO_D: + case INS_OPTS_S_TO_D: + case INS_OPTS_H_TO_D: - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); + return EA_8BYTE; - dispIns(id); - appendToCurIG(id); -} + case INS_OPTS_S_TO_4BYTE: + case INS_OPTS_D_TO_4BYTE: + case INS_OPTS_4BYTE_TO_S: + case INS_OPTS_8BYTE_TO_S: + case INS_OPTS_D_TO_S: + case INS_OPTS_H_TO_S: -/***************************************************************************** - * - * Add an instruction with a static data member operand. If 'size' is 0, the - * instruction operates on the address of the static member instead of its - * value (e.g. "push offset clsvar", rather than "push dword ptr [clsvar]"). - */ + return EA_4BYTE; -void emitter::emitIns_C(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE fldHnd, int offs) -{ - NYI("emitIns_C"); -} + case INS_OPTS_S_TO_H: + case INS_OPTS_D_TO_H: -/***************************************************************************** - * - * Add an instruction referencing stack-based local variable. - */ + return EA_2BYTE; -void emitter::emitIns_S(instruction ins, emitAttr attr, int varx, int offs) -{ - NYI("emitIns_S"); + default: + assert(!" invalid 'conversion' value"); + return EA_UNKNOWN; + } } -/***************************************************************************** - * - * Add an instruction referencing a register and a stack-based local variable. - */ -void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int varx, int offs) +// For the given 'conversion' returns the 'srcsize' specified by the conversion option +/*static*/ emitAttr emitter::optGetSrcsize(insOpts conversion) { - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - int disp = 0; - unsigned scale = 0; - bool isLdrStr = false; - - assert(offs >= 0); - - // TODO-ARM64-CQ: use unscaled loads? - /* Figure out the encoding format of the instruction */ - switch (ins) + switch (conversion) { - case INS_strb: - case INS_ldrb: - case INS_ldrsb: - scale = 0; - break; - - case INS_strh: - case INS_ldrh: - case INS_ldrsh: - scale = 1; - break; - - case INS_ldrsw: - scale = 2; - break; - - case INS_str: - case INS_ldr: - assert(isValidGeneralDatasize(size) || isValidVectorDatasize(size)); - scale = genLog2(EA_SIZE_IN_BYTES(size)); - isLdrStr = true; - break; + case INS_OPTS_D_TO_8BYTE: + case INS_OPTS_D_TO_4BYTE: + case INS_OPTS_8BYTE_TO_D: + case INS_OPTS_8BYTE_TO_S: + case INS_OPTS_D_TO_S: + case INS_OPTS_D_TO_H: - case INS_lea: - assert(size == EA_8BYTE); - scale = 0; - break; + return EA_8BYTE; - default: - NYI("emitIns_R_S"); // FP locals? - return; + case INS_OPTS_S_TO_8BYTE: + case INS_OPTS_S_TO_4BYTE: + case INS_OPTS_4BYTE_TO_S: + case INS_OPTS_4BYTE_TO_D: + case INS_OPTS_S_TO_D: + case INS_OPTS_S_TO_H: - } // end switch (ins) + return EA_4BYTE; - /* Figure out the variable's frame position */ - ssize_t imm; - int base; - bool FPbased; + case INS_OPTS_H_TO_S: + case INS_OPTS_H_TO_D: - base = emitComp->lvaFrameAddress(varx, &FPbased); - disp = base + offs; - assert((scale >= 0) && (scale <= 4)); + return EA_2BYTE; - bool useRegForImm = false; - regNumber reg2 = FPbased ? REG_FPBASE : REG_SPBASE; - reg2 = encodingSPtoZR(reg2); + default: + assert(!" invalid 'conversion' value"); + return EA_UNKNOWN; + } +} - if (ins == INS_lea) - { - if (disp >= 0) - { - ins = INS_add; - imm = disp; - } - else - { - ins = INS_sub; - imm = -disp; - } +// For the given 'size' and 'index' returns true if it specifies a valid index for a vector register of 'size' +/*static*/ bool emitter::isValidVectorIndex(emitAttr datasize, emitAttr elemsize, ssize_t index) +{ + assert(isValidVectorDatasize(datasize)); + assert(isValidVectorElemsize(elemsize)); - if (imm <= 0x0fff) - { - fmt = IF_DI_2A; // add reg1,reg2,#disp - } - else - { - regNumber rsvdReg = codeGen->rsGetRsvdReg(); - codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); - fmt = IF_DR_3A; // add reg1,reg2,rsvdReg - } - } - else + bool result = false; + if (index >= 0) { - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - imm = disp; - if (imm == 0) - { - fmt = IF_LS_2A; - } - else if ((imm < 0) || ((imm & mask) != 0)) + if (datasize == EA_8BYTE) { - if ((imm >= -256) && (imm <= 255)) - { - fmt = IF_LS_2C; - } - else + switch (elemsize) { - useRegForImm = true; + case EA_1BYTE: + result = (index < 8); + break; + case EA_2BYTE: + result = (index < 4); + break; + case EA_4BYTE: + result = (index < 2); + break; + case EA_8BYTE: + result = (index < 1); + break; + default: + unreached(); + break; } } - else if (imm > 0) + else if (datasize == EA_16BYTE) { - if (((imm & mask) == 0) && ((imm >> scale) < 0x1000)) - { - imm >>= scale; // The immediate is scaled by the size of the ld/st - - fmt = IF_LS_2B; - } - else + switch (elemsize) { - useRegForImm = true; + case EA_1BYTE: + result = (index < 16); + break; + case EA_2BYTE: + result = (index < 8); + break; + case EA_4BYTE: + result = (index < 4); + break; + case EA_8BYTE: + result = (index < 2); + break; + default: + unreached(); + break; } } - - if (useRegForImm) - { - regNumber rsvdReg = codeGen->rsGetRsvdReg(); - codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); - fmt = IF_LS_3A; - } } + return result; +} - assert(fmt != IF_NONE); +/***************************************************************************** + * + * Add an instruction with no operands. + */ - // Try to optimize a load/store with an alternative instruction. - if (isLdrStr && emitComp->opts.OptimizationEnabled() && - OptimizeLdrStr(ins, attr, reg1, reg2, imm, size, fmt, true, varx, offs DEBUG_ARG(useRegForImm))) +void emitter::emitIns(instruction ins) +{ + instrDesc* id = emitNewInstrSmall(EA_8BYTE); + insFormat fmt = emitInsFormat(ins); + + if (ins != INS_BREAKPOINT) { - return; + assert(fmt == IF_SN_0A); } - instrDesc* id = emitNewInstrCns(attr, imm); - id->idIns(ins); id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - - id->idReg1(reg1); - id->idReg2(reg2); - id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs); - id->idSetIsLclVar(); - -#ifdef DEBUG - id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs; -#endif dispIns(id); appendToCurIG(id); @@ -11901,94 +5654,81 @@ void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int va /***************************************************************************** * - * Add an instruction referencing two register and consecutive stack-based local variable slots. + * Add an instruction with a single immediate value. */ -void emitter::emitIns_R_R_S_S( - instruction ins, emitAttr attr1, emitAttr attr2, regNumber reg1, regNumber reg2, int varx, int offs) -{ - assert((ins == INS_ldp) || (ins == INS_ldnp)); - assert(EA_8BYTE == EA_SIZE(attr1)); - assert(EA_8BYTE == EA_SIZE(attr2)); - assert(isGeneralRegisterOrZR(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert(offs >= 0); - - insFormat fmt = IF_LS_3B; - int disp = 0; - const unsigned scale = 3; - /* Figure out the variable's frame position */ - int base; - bool FPbased; - - base = emitComp->lvaFrameAddress(varx, &FPbased); - disp = base + offs; - - // TODO-ARM64-CQ: with compLocallocUsed, should we use REG_SAVED_LOCALLOC_SP instead? - regNumber reg3 = FPbased ? REG_FPBASE : REG_SPBASE; - reg3 = encodingSPtoZR(reg3); +void emitter::emitIns_I(instruction ins, emitAttr attr, ssize_t imm) +{ + insFormat fmt = IF_NONE; - bool useRegForAdr = true; - ssize_t imm = disp; - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - if (imm == 0) - { - useRegForAdr = false; - } - else + /* Figure out the encoding format of the instruction */ + if (ins == INS_BREAKPOINT) { - if ((imm & mask) == 0) + if ((imm & 0x0000ffff) == imm) { - ssize_t immShift = imm >> scale; // The immediate is scaled by the size of the ld/st - - if ((immShift >= -64) && (immShift <= 63)) - { - fmt = IF_LS_3C; - useRegForAdr = false; - imm = immShift; - } + fmt = IF_SI_0A; + } + else + { + assert(!"Instruction cannot be encoded: IF_SI_0A"); } } - - if (useRegForAdr) + else { - regNumber rsvd = codeGen->rsGetRsvdReg(); - emitIns_R_R_Imm(INS_add, EA_PTRSIZE, rsvd, reg3, imm); - reg3 = rsvd; - imm = 0; + // fallback to emit SVE instructions. + return emitInsSve_I(ins, attr, imm); } - assert(fmt != IF_NONE); - instrDesc* id = emitNewInstrCns(attr1, imm); + instrDesc* id = emitNewInstrSC(attr, imm); id->idIns(ins); id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - // Record the attribute for the second register in the pair - if (EA_IS_GCREF(attr2)) - { - id->idGCrefReg2(GCT_GCREF); - } - else if (EA_IS_BYREF(attr2)) - { - id->idGCrefReg2(GCT_BYREF); - } - else + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add an instruction referencing a single register. + */ + +void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg, insOpts opt /* = INS_OPTS_NONE */) +{ + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) { - id->idGCrefReg2(GCT_NONE); + case INS_br: + case INS_ret: + assert(isGeneralRegister(reg)); + fmt = IF_BR_1A; + break; + + case INS_dczva: + assert(isGeneralRegister(reg)); + assert(attr == EA_8BYTE); + fmt = IF_SR_1A; + break; + + case INS_mrs_tpid0: + fmt = IF_SR_1A; + break; + + default: + // fallback to emit SVE instructions. + return emitInsSve_R(ins, attr, reg, opt); } - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs); - id->idSetIsLclVar(); + assert(fmt != IF_NONE); -#ifdef DEBUG - id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs; -#endif + instrDesc* id = emitNewInstrSmall(attr); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idReg1(reg); dispIns(id); appendToCurIG(id); @@ -11996,233 +5736,268 @@ void emitter::emitIns_R_R_S_S( /***************************************************************************** * - * Add an instruction referencing a stack-based local variable and a register + * Add an instruction referencing a register and a constant. */ -void emitter::emitIns_S_R(instruction ins, emitAttr attr, regNumber reg1, int varx, int offs) + +void emitter::emitIns_R_I(instruction ins, + emitAttr attr, + regNumber reg, + ssize_t imm, + insOpts opt, /* = INS_OPTS_NONE */ + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */ + DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = GTF_EMPTY */)) { - assert(offs >= 0); - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - int disp = 0; - unsigned scale = 0; - bool isVectorStore = false; - bool isStr = false; + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; + bool canEncode = false; - // TODO-ARM64-CQ: use unscaled loads? /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_strb: - scale = 0; - assert(isGeneralRegisterOrZR(reg1)); - break; - - case INS_strh: - scale = 1; - assert(isGeneralRegisterOrZR(reg1)); - break; + bitMaskImm bmi; + halfwordImm hwi; + byteShiftedImm bsi; + ssize_t notOfImm; - case INS_str: - if (isGeneralRegisterOrZR(reg1)) - { - assert(isValidGeneralDatasize(size)); - scale = (size == EA_8BYTE) ? 3 : 2; - } - else + case INS_tst: + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg)); + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, size, &bmi); + if (canEncode) { - assert(isVectorRegister(reg1)); - assert(isValidVectorLSDatasize(size)); - scale = NaturalScale_helper(size); - isVectorStore = true; + imm = bmi.immNRS; + assert(isValidImmNRS(imm, size)); + fmt = IF_DI_1C; } - isStr = true; break; - default: - NYI("emitIns_S_R"); // FP locals? - return; - - } // end switch (ins) - - /* Figure out the variable's frame position */ - int base; - bool FPbased; - - base = emitComp->lvaFrameAddress(varx, &FPbased); - disp = base + offs; - assert(scale >= 0); - if (isVectorStore) - { - assert(scale <= 4); - } - else - { - assert(scale <= 3); - } + case INS_movk: + case INS_movn: + case INS_movz: + assert(isValidGeneralDatasize(size)); + assert(insOptsNone(opt)); // No LSL here (you must use emitIns_R_I_I if a shift is needed) + assert(isGeneralRegister(reg)); + assert(isValidUimm<16>(imm)); - // TODO-ARM64-CQ: with compLocallocUsed, should we use REG_SAVED_LOCALLOC_SP instead? - regNumber reg2 = FPbased ? REG_FPBASE : REG_SPBASE; - reg2 = encodingSPtoZR(reg2); + hwi.immHW = 0; + hwi.immVal = imm; + assert(imm == emitDecodeHalfwordImm(hwi, size)); - bool useRegForImm = false; - ssize_t imm = disp; - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - if (imm == 0) - { - fmt = IF_LS_2A; - } - else if ((imm < 0) || ((imm & mask) != 0)) - { - if ((imm >= -256) && (imm <= 255)) - { - fmt = IF_LS_2C; - } - else - { - useRegForImm = true; - } - } - else if (imm > 0) - { - if (((imm & mask) == 0) && ((imm >> scale) < 0x1000)) - { - imm >>= scale; // The immediate is scaled by the size of the ld/st + imm = hwi.immHWVal; + canEncode = true; + fmt = IF_DI_1B; + break; - fmt = IF_LS_2B; - } - else - { - useRegForImm = true; - } - } + case INS_mov: + assert(isValidGeneralDatasize(size)); + assert(insOptsNone(opt)); // No explicit LSL here + // We will automatically determine the shift based upon the imm - if (useRegForImm) - { - // The reserved register is not stored in idReg3() since that field overlaps with iiaLclVar. - // It is instead implicit when idSetIsLclVar() is set, with this encoding format. - regNumber rsvdReg = codeGen->rsGetRsvdReg(); - codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); - fmt = IF_LS_3A; - } + // First try the standard 'halfword immediate' imm(i16,hw) + hwi.immHWVal = 0; + canEncode = canEncodeHalfwordImm(imm, size, &hwi); + if (canEncode) + { + // uses a movz encoding + assert(isGeneralRegister(reg)); + imm = hwi.immHWVal; + assert(isValidImmHWVal(imm, size)); + fmt = IF_DI_1B; + break; + } - assert(fmt != IF_NONE); + // Next try the ones-complement form of 'halfword immediate' imm(i16,hw) + notOfImm = NOT_helper(imm, getBitWidth(size)); + canEncode = canEncodeHalfwordImm(notOfImm, size, &hwi); + if (canEncode) + { + assert(isGeneralRegister(reg)); + imm = hwi.immHWVal; + ins = INS_movn; // uses a movn encoding + assert(isValidImmHWVal(imm, size)); + fmt = IF_DI_1B; + break; + } - // Try to optimize a store with an alternative instruction. - if (isStr && emitComp->opts.OptimizationEnabled() && - OptimizeLdrStr(ins, attr, reg1, reg2, imm, size, fmt, true, varx, offs DEBUG_ARG(useRegForImm))) - { - return; - } + // Finally try the 'bitmask immediate' imm(N,r,s) + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, size, &bmi); + if (canEncode) + { + assert(isGeneralRegisterOrSP(reg)); + reg = encodingSPtoZR(reg); + imm = bmi.immNRS; + assert(isValidImmNRS(imm, size)); + fmt = IF_DI_1D; + break; + } + else + { + assert(!"Instruction cannot be encoded: mov imm"); + } - instrDesc* id = emitNewInstrCns(attr, imm); + break; - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); + case INS_movi: + assert(isValidVectorDatasize(size)); + assert(isVectorRegister(reg)); + if (insOptsNone(opt) && (size == EA_8BYTE)) + { + opt = INS_OPTS_1D; + } + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); - id->idReg1(reg1); - id->idReg2(reg2); - id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs); - id->idSetIsLclVar(); + if (elemsize == EA_8BYTE) + { + size_t uimm = imm; + ssize_t imm8 = 0; + unsigned pos = 0; + canEncode = true; + while (uimm != 0) + { + INT64 loByte = uimm & 0xFF; + if (((loByte == 0) || (loByte == 0xFF)) && (pos < 8)) + { + if (loByte == 0xFF) + { + imm8 |= (ssize_t{1} << pos); + } + uimm >>= 8; + pos++; + } + else + { + canEncode = false; + break; + } + } + imm = imm8; + assert(isValidUimm<8>(imm)); + fmt = IF_DV_1B; + break; + } + else + { + // Vector operation -#ifdef DEBUG - id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs; -#endif + // No explicit LSL/MSL is used for the immediate + // We will automatically determine the shift based upon the value of imm - dispIns(id); - appendToCurIG(id); -} + // First try the standard 'byteShifted immediate' imm(i8,bySh) + bsi.immBSVal = 0; + canEncode = canEncodeByteShiftedImm(imm, elemsize, true, &bsi); + if (canEncode) + { + imm = bsi.immBSVal; + assert(isValidImmBSVal(imm, size)); + fmt = IF_DV_1B; + break; + } -/***************************************************************************** - * - * Add an instruction referencing consecutive stack-based local variable slots and two registers - */ -void emitter::emitIns_S_S_R_R( - instruction ins, emitAttr attr1, emitAttr attr2, regNumber reg1, regNumber reg2, int varx, int offs) -{ - assert((ins == INS_stp) || (ins == INS_stnp)); - assert(EA_8BYTE == EA_SIZE(attr1)); - assert(EA_8BYTE == EA_SIZE(attr2)); - assert(isGeneralRegisterOrZR(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert(offs >= 0); + // Next try the ones-complement form of the 'immediate' imm(i8,bySh) + if ((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)) // Only EA_2BYTE or EA_4BYTE forms + { + notOfImm = NOT_helper(imm, getBitWidth(elemsize)); + canEncode = canEncodeByteShiftedImm(notOfImm, elemsize, true, &bsi); + if (canEncode) + { + imm = bsi.immBSVal; + ins = INS_mvni; // uses a mvni encoding + assert(isValidImmBSVal(imm, size)); + fmt = IF_DV_1B; + break; + } + } + } + break; - insFormat fmt = IF_LS_3B; - int disp = 0; - const unsigned scale = 3; + case INS_orr: + case INS_bic: + case INS_mvni: + assert(isValidVectorDatasize(size)); + assert(isVectorRegister(reg)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)); // Only EA_2BYTE or EA_4BYTE forms - /* Figure out the variable's frame position */ - int base; - bool FPbased; + // Vector operation - base = emitComp->lvaFrameAddress(varx, &FPbased); - disp = base + offs; + // No explicit LSL/MSL is used for the immediate + // We will automatically determine the shift based upon the value of imm - // TODO-ARM64-CQ: with compLocallocUsed, should we use REG_SAVED_LOCALLOC_SP instead? - regNumber reg3 = FPbased ? REG_FPBASE : REG_SPBASE; + // First try the standard 'byteShifted immediate' imm(i8,bySh) + bsi.immBSVal = 0; + canEncode = canEncodeByteShiftedImm(imm, elemsize, + (ins == INS_mvni), // mvni supports the ones shifting variant (aka MSL) + &bsi); + if (canEncode) + { + imm = bsi.immBSVal; + assert(isValidImmBSVal(imm, size)); + fmt = IF_DV_1B; + break; + } + break; - bool useRegForAdr = true; - ssize_t imm = disp; - ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate - if (imm == 0) - { - useRegForAdr = false; - } - else - { - if ((imm & mask) == 0) - { - ssize_t immShift = imm >> scale; // The immediate is scaled by the size of the ld/st + case INS_cmp: + case INS_cmn: + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg)); - if ((immShift >= -64) && (immShift <= 63)) + if (unsigned_abs(imm) <= 0x0fff) { - fmt = IF_LS_3C; - useRegForAdr = false; - imm = immShift; + if (imm < 0) + { + ins = insReverse(ins); + imm = -imm; + } + assert(isValidUimm<12>(imm)); + canEncode = true; + fmt = IF_DI_1A; } - } - } + else if (canEncodeWithShiftImmBy12(imm)) // Try the shifted by 12 encoding + { + // Encoding will use a 12-bit left shift of the immediate + opt = INS_OPTS_LSL12; + if (imm < 0) + { + ins = insReverse(ins); + imm = -imm; + } + assert((imm & 0xfff) == 0); + imm >>= 12; + assert(isValidUimm<12>(imm)); + canEncode = true; + fmt = IF_DI_1A; + } + else + { + assert(!"Instruction cannot be encoded: IF_DI_1A"); + } + break; - if (useRegForAdr) - { - regNumber rsvd = codeGen->rsGetRsvdReg(); - emitIns_R_R_Imm(INS_add, EA_PTRSIZE, rsvd, reg3, imm); - reg3 = rsvd; - imm = 0; - } + default: + // fallback to emit SVE instructions. + return emitInsSve_R_I(ins, attr, reg, imm, opt, sopt); + } // end switch (ins) + assert(canEncode); assert(fmt != IF_NONE); - instrDesc* id = emitNewInstrCns(attr1, imm); + instrDesc* id = emitNewInstrSC(attr, imm); id->idIns(ins); id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - - // Record the attribute for the second register in the pair - if (EA_IS_GCREF(attr2)) - { - id->idGCrefReg2(GCT_GCREF); - } - else if (EA_IS_BYREF(attr2)) - { - id->idGCrefReg2(GCT_BYREF); - } - else - { - id->idGCrefReg2(GCT_NONE); - } - - reg3 = encodingSPtoZR(reg3); + id->idInsOpt(opt); - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs); - id->idSetIsLclVar(); + id->idReg1(reg); #ifdef DEBUG - id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs; + id->idDebugOnlyInfo()->idMemCookie = targetHandle; + id->idDebugOnlyInfo()->idFlags = gtFlags; #endif dispIns(id); @@ -12231,8601 +6006,8463 @@ void emitter::emitIns_S_S_R_R( /***************************************************************************** * - * Add an instruction referencing stack-based local variable and an immediate + * Add an instruction referencing a register and a floating point constant. */ -void emitter::emitIns_S_I(instruction ins, emitAttr attr, int varx, int offs, int val) -{ - NYI("emitIns_S_I"); -} -/***************************************************************************** - * - * Add an instruction with a register + static member operands. - * Constant is stored into JIT data which is adjacent to code. - * No relocation is needed. PC-relative offset will be encoded directly into instruction. - * - */ -void emitter::emitIns_R_C( - instruction ins, emitAttr attr, regNumber reg, regNumber addrReg, CORINFO_FIELD_HANDLE fldHnd, int offs) -{ - assert(offs >= 0); - assert(instrDesc::fitsInSmallCns(offs)); +void emitter::emitIns_R_F( + instruction ins, emitAttr attr, regNumber reg, double immDbl, insOpts opt /* = INS_OPTS_NONE */) - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - instrDescJmp* id = emitNewInstrJmp(); +{ + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; + ssize_t imm = 0; + bool canEncode = false; + /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_adr: - // This is case to get address to the constant data. - fmt = IF_LARGEADR; - assert(isGeneralRegister(reg)); - assert(isValidGeneralDatasize(size)); + floatImm8 fpi; + + case INS_fcmp: + case INS_fcmpe: + assert(insOptsNone(opt)); + assert(isValidVectorElemsizeFloat(size)); + assert(isVectorRegister(reg)); + if (immDbl == 0.0) + { + canEncode = true; + fmt = IF_DV_1C; + } break; - case INS_ldr: - fmt = IF_LARGELDC; - if (isVectorRegister(reg)) + case INS_fmov: + assert(isVectorRegister(reg)); + fpi.immFPIVal = 0; + canEncode = canEncodeFloatImm8(immDbl, &fpi); + + if (insOptsAnyArrangement(opt)) { - assert(isValidVectorLSDatasize(size)); - // For vector (float/double) register, we should have an integer address reg to - // compute long address which consists of page address and page offset. - // For integer constant, this is not needed since the dest reg can be used to - // compute address as well as contain the final contents. - assert(isGeneralRegister(reg) || (addrReg != REG_NA)); + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsizeFloat(elemsize)); + assert(opt != INS_OPTS_1D); // Reserved encoding + + if (canEncode) + { + imm = fpi.immFPIVal; + assert((imm >= 0) && (imm <= 0xff)); + fmt = IF_DV_1B; + } } else { - assert(isGeneralRegister(reg)); - assert(isValidGeneralDatasize(size)); + // Scalar operation + assert(insOptsNone(opt)); + assert(isValidVectorElemsizeFloat(size)); + + if (canEncode) + { + imm = fpi.immFPIVal; + assert((imm >= 0) && (imm <= 0xff)); + fmt = IF_DV_1A; + } } break; - default: - unreached(); - } + default: + // fallback to emit SVE instructions. + return emitInsSve_R_F(ins, attr, reg, immDbl, opt); + + } // end switch (ins) + assert(canEncode); assert(fmt != IF_NONE); + instrDesc* id = emitNewInstrSC(attr, imm); + id->idIns(ins); id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - id->idSmallCns(offs); - id->idOpSize(size); - id->idAddr()->iiaFieldHnd = fldHnd; - id->idSetIsBound(); // We won't patch address since we will know the exact distance once JIT code and data are - // allocated together. - - id->idReg1(reg); // destination register that will get the constant value. - if (addrReg != REG_NA) - { - id->idReg2(addrReg); // integer register to compute long address (used for vector dest when we end up with long - // address) - } - id->idjShort = false; // Assume loading constant from long address - - // Keep it long if it's in cold code. - id->idjKeepLong = emitComp->fgIsBlockCold(emitComp->compCurBB); - -#ifdef DEBUG - if (emitComp->opts.compLongAddress) - id->idjKeepLong = 1; -#endif // DEBUG - - // If it's possible to be shortened, then put it in jump list - // to be revisited by emitJumpDistBind. - if (!id->idjKeepLong) - { - /* Record the jump's IG and offset within it */ - id->idjIG = emitCurIG; - id->idjOffs = emitCurIGsize; - - /* Append this jump to this IG's jump list */ - id->idjNext = emitCurIGjmpList; - emitCurIGjmpList = id; + id->idInsOpt(opt); -#if EMITTER_STATS - emitTotalIGjmps++; -#endif - } + id->idReg1(reg); dispIns(id); appendToCurIG(id); } -/***************************************************************************** - * - * Add an instruction with a static member + constant. - */ - -void emitter::emitIns_C_I(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE fldHnd, ssize_t offs, ssize_t val) +//------------------------------------------------------------------------ +// emitIns_Mov: Emits a move instruction +// +// Arguments: +// ins -- The instruction being emitted +// attr -- The emit attribute +// dstReg -- The destination register +// srcReg -- The source register +// canSkip -- true if the move can be elided when dstReg == srcReg, otherwise false +// insOpts -- The instruction options +// +void emitter::emitIns_Mov( + instruction ins, emitAttr attr, regNumber dstReg, regNumber srcReg, bool canSkip, insOpts opt /* = INS_OPTS_NONE */) { - NYI("emitIns_C_I"); -} + assert(IsMovInstruction(ins)); -/***************************************************************************** - * - * Add an instruction with a static member + register operands. - */ + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; -void emitter::emitIns_C_R(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE fldHnd, regNumber reg, int offs) -{ - assert(!"emitIns_C_R not supported for RyuJIT backend"); -} + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_mov: + { + assert(insOptsNone(opt)); -void emitter::emitIns_R_AR(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, int offs) -{ - NYI("emitIns_R_AR"); -} + if (IsRedundantMov(ins, size, dstReg, srcReg, canSkip)) + { + // These instructions have no side effect and can be skipped + return; + } -// This generates code to populate the access for TLS on linux -void emitter::emitIns_Adrp_Ldr_Add(emitAttr attr, - regNumber reg1, - regNumber reg2, - ssize_t addr DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) -{ - assert(emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI)); - assert(TargetOS::IsUnix); - assert(EA_IS_RELOC(attr)); - assert(EA_IS_CNS_TLSGD_RELOC(attr)); + // Check for the 'mov' aliases for the vector registers + if (isVectorRegister(dstReg)) + { + if (isVectorRegister(srcReg) && isValidVectorDatasize(size)) + { + return emitIns_R_R_R(INS_mov, size, dstReg, srcReg, srcReg); + } + else + { + return emitIns_R_R_I(INS_mov, size, dstReg, srcReg, 0); + } + } + else + { + if (isVectorRegister(srcReg)) + { + assert(isGeneralRegister(dstReg)); + return emitIns_R_R_I(INS_mov, size, dstReg, srcReg, 0); + } + } - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_DI_1E; - bool needAdd = false; - instrDescJmp* id = emitNewInstrJmp(); + // Is this a MOV to/from SP instruction? + if ((dstReg == REG_SP) || (srcReg == REG_SP)) + { + assert(isGeneralRegisterOrSP(dstReg)); + assert(isGeneralRegisterOrSP(srcReg)); + dstReg = encodingSPtoZR(dstReg); + srcReg = encodingSPtoZR(srcReg); + fmt = IF_DR_2G; + } + else + { + assert(insOptsNone(opt)); + assert(isGeneralRegister(dstReg)); + assert(isGeneralRegisterOrZR(srcReg)); + fmt = IF_DR_2E; + } + break; + } - // adrp - id->idIns(INS_adrp); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - id->idOpSize(size); - id->idAddr()->iiaAddr = (BYTE*)addr; - id->idReg1(reg1); - id->idSetIsDspReloc(); - id->idSetTlsGD(); + case INS_sxtw: + { + assert((size == EA_8BYTE) || (size == EA_4BYTE)); + FALLTHROUGH; + } -#ifdef DEBUG - id->idDebugOnlyInfo()->idMemCookie = targetHandle; - id->idDebugOnlyInfo()->idFlags = gtFlags; -#endif + case INS_sxtb: + case INS_sxth: + case INS_uxtb: + case INS_uxth: + { + if (canSkip && (dstReg == srcReg)) + { + // There are scenarios such as in genCallInstruction where the sign/zero extension should be elided + return; + } - dispIns(id); - appendToCurIG(id); + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(dstReg)); + assert(isGeneralRegister(srcReg)); + fmt = IF_DR_2H; + break; + } - // ldr - emitIns_R_R_I(INS_ldr, attr, reg2, reg1, (ssize_t)addr); + case INS_fmov: + { + assert(isValidVectorElemsizeFloat(size)); - // add - fmt = IF_DI_2A; - instrDesc* addId = emitNewInstr(attr); - assert(id->idIsReloc()); + if (canSkip && (dstReg == srcReg)) + { + // These instructions have no side effect and can be skipped + return; + } - addId->idIns(INS_add); - addId->idInsFmt(fmt); - addId->idInsOpt(INS_OPTS_NONE); - addId->idOpSize(size); - addId->idAddr()->iiaAddr = (BYTE*)addr; - addId->idReg1(reg1); - addId->idReg2(reg1); - addId->idSetTlsGD(); + if (isVectorRegister(dstReg)) + { + if (isVectorRegister(srcReg)) + { + assert(insOptsNone(opt)); + fmt = IF_DV_2G; + } + else + { + assert(isGeneralRegister(srcReg)); - dispIns(addId); - appendToCurIG(addId); -} + // if the optional conversion specifier is not present we calculate it + if (opt == INS_OPTS_NONE) + { + opt = (size == EA_4BYTE) ? INS_OPTS_4BYTE_TO_S : INS_OPTS_8BYTE_TO_D; + } + assert(insOptsConvertIntToFloat(opt)); -// This computes address from the immediate which is relocatable. -void emitter::emitIns_R_AI(instruction ins, - emitAttr attr, - regNumber ireg, - ssize_t addr DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) -{ - assert(EA_IS_RELOC(attr)); - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_DI_1E; - bool needAdd = false; - instrDescJmp* id = emitNewInstrJmp(); + fmt = IF_DV_2I; + } + } + else + { + assert(isGeneralRegister(dstReg)); + assert(isVectorRegister(srcReg)); - switch (ins) - { - case INS_adrp: - // This computes page address. - // page offset is needed using add. - needAdd = true; - break; - case INS_adr: + // if the optional conversion specifier is not present we calculate it + if (opt == INS_OPTS_NONE) + { + opt = (size == EA_4BYTE) ? INS_OPTS_S_TO_4BYTE : INS_OPTS_D_TO_8BYTE; + } + assert(insOptsConvertFloatToInt(opt)); + + fmt = IF_DV_2H; + } break; + } + default: + { unreached(); + } } + assert(fmt != IF_NONE); + + instrDesc* id = emitNewInstrSmall(attr); + id->idIns(ins); id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - id->idOpSize(size); - id->idAddr()->iiaAddr = (BYTE*)addr; - id->idReg1(ireg); - id->idSetIsDspReloc(); -#ifdef DEBUG - id->idDebugOnlyInfo()->idMemCookie = targetHandle; - id->idDebugOnlyInfo()->idFlags = gtFlags; -#endif + id->idInsOpt(opt); + + id->idReg1(dstReg); + id->idReg2(srcReg); dispIns(id); appendToCurIG(id); +} - if (needAdd) +/***************************************************************************** + * + * Add an instruction referencing two registers + */ + +void emitter::emitIns_R_R(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +{ + if (IsMovInstruction(ins)) { - // add reg, reg, imm - ins = INS_add; - fmt = IF_DI_2A; - instrDesc* id = emitNewInstr(attr); - assert(id->idIsReloc()); + assert(!"Please use emitIns_Mov() to correctly handle move elision"); + emitIns_Mov(ins, attr, reg1, reg2, /* canSkip */ false, opt); + } - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(INS_OPTS_NONE); - id->idOpSize(size); - id->idAddr()->iiaAddr = (BYTE*)addr; - id->idReg1(ireg); - id->idReg2(ireg); + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; - dispIns(id); - appendToCurIG(id); - } -} + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_dup: + // Vector operation + assert(insOptsAnyArrangement(opt)); + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_2C; + break; + + case INS_abs: + case INS_not: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + if (ins == INS_not) + { + assert(isValidVectorDatasize(size)); + // Bitwise behavior is independent of element size, but is always encoded as 1 Byte + opt = optMakeArrangement(size, EA_1BYTE); + } + if (insOptsNone(opt)) + { + // Scalar operation + assert(size == EA_8BYTE); // Only type D is supported + fmt = IF_DV_2L; + } + else + { + // Vector operation + assert(insOptsAnyArrangement(opt)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + fmt = IF_DV_2M; + } + break; + + case INS_mvn: + case INS_neg: + if (isVectorRegister(reg1)) + { + assert(isVectorRegister(reg2)); + if (ins == INS_mvn) + { + assert(isValidVectorDatasize(size)); + // Bitwise behavior is independent of element size, but is always encoded as 1 Byte + opt = optMakeArrangement(size, EA_1BYTE); + } + if (insOptsNone(opt)) + { + // Scalar operation + assert(size == EA_8BYTE); // Only type D is supported + fmt = IF_DV_2L; + } + else + { + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + fmt = IF_DV_2M; + } + break; + } + FALLTHROUGH; + + case INS_negs: + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + fmt = IF_DR_2E; + break; + + case INS_sxtl: + case INS_sxtl2: + case INS_uxtl: + case INS_uxtl2: + return emitIns_R_R_I(ins, size, reg1, reg2, 0, opt); + + case INS_cls: + case INS_clz: + case INS_rbit: + case INS_rev16: + case INS_rev32: + case INS_cnt: + if (isVectorRegister(reg1)) + { + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + if ((ins == INS_cls) || (ins == INS_clz)) + { + assert(elemsize != EA_8BYTE); // No encoding for type D + } + else if (ins == INS_rev32) + { + assert((elemsize == EA_2BYTE) || (elemsize == EA_1BYTE)); + } + else + { + assert(elemsize == EA_1BYTE); // Only supports 8B or 16B + } + fmt = IF_DV_2M; + break; + } + if (ins == INS_cnt) + { + // Doesn't have general register version(s) + break; + } + + FALLTHROUGH; + + case INS_rev: + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + if (ins == INS_rev32) + { + assert(size == EA_8BYTE); + } + else + { + assert(isValidGeneralDatasize(size)); + } + fmt = IF_DR_2G; + break; + + case INS_addv: + case INS_saddlv: + case INS_smaxv: + case INS_sminv: + case INS_uaddlv: + case INS_umaxv: + case INS_uminv: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + assert((opt != INS_OPTS_2S) && (opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // Reserved encodings + fmt = IF_DV_2T; + break; + + case INS_rev64: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(elemsize != EA_8BYTE); // No encoding for type D + fmt = IF_DV_2M; + break; + + case INS_sqxtn: + case INS_sqxtun: + case INS_uqxtn: + if (insOptsNone(opt)) + { + // Scalar operation + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorElemsize(size)); + assert(size != EA_8BYTE); // The encoding size = 11 is reserved. + fmt = IF_DV_2L; + break; + } + FALLTHROUGH; + + case INS_xtn: + // Vector operation + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(size == EA_8BYTE); + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = x is reserved + fmt = IF_DV_2M; + break; + + case INS_sqxtn2: + case INS_sqxtun2: + case INS_uqxtn2: + case INS_xtn2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(size == EA_16BYTE); + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_2D); // The encoding size = 11, Q = x is reserved + fmt = IF_DV_2M; + break; -void emitter::emitIns_AR_R(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, int offs) -{ - NYI("emitIns_AR_R"); -} + case INS_ldar: + case INS_ldapr: + case INS_ldaxr: + case INS_ldxr: + case INS_stlr: + assert(isValidGeneralDatasize(size)); -void emitter::emitIns_R_ARR(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, regNumber rg2, int disp) -{ - NYI("emitIns_R_ARR"); -} + FALLTHROUGH; -void emitter::emitIns_ARR_R(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, regNumber rg2, int disp) -{ - NYI("emitIns_R_ARR"); -} + case INS_ldarb: + case INS_ldaprb: + case INS_ldaxrb: + case INS_ldxrb: + case INS_ldarh: + case INS_ldaprh: + case INS_ldaxrh: + case INS_ldxrh: + case INS_stlrb: + case INS_stlrh: + assert(isValidGeneralLSDatasize(size)); + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrSP(reg2)); + assert(insOptsNone(opt)); -void emitter::emitIns_R_ARX( - instruction ins, emitAttr attr, regNumber ireg, regNumber reg, regNumber rg2, unsigned mul, int disp) -{ - NYI("emitIns_R_ARR"); -} + reg2 = encodingSPtoZR(reg2); -/***************************************************************************** - * - * Record that a jump instruction uses the short encoding - * - */ -void emitter::emitSetShortJump(instrDescJmp* id) -{ - if (id->idjKeepLong) - return; + fmt = IF_LS_2A; + break; - insFormat fmt = IF_NONE; - if (emitIsCondJump(id)) - { - switch (id->idIns()) - { - case INS_cbz: - case INS_cbnz: - fmt = IF_BI_1A; - break; - case INS_tbz: - case INS_tbnz: - fmt = IF_BI_1B; - break; - default: - fmt = IF_BI_0B; - break; - } - } - else if (emitIsLoadLabel(id)) - { - fmt = IF_DI_1E; - } - else if (emitIsLoadConstant(id)) - { - fmt = IF_LS_1A; - } - else - { - unreached(); - } + case INS_ldr: + case INS_ldrb: + case INS_ldrh: + case INS_ldrsb: + case INS_ldrsh: + case INS_ldrsw: + case INS_str: + case INS_strb: + case INS_strh: + case INS_cmn: + case INS_tst: + assert(insOptsNone(opt)); + emitIns_R_R_I(ins, attr, reg1, reg2, 0, INS_OPTS_NONE); + return; - id->idInsFmt(fmt); - id->idjShort = true; -} + case INS_cmp: + emitIns_R_R_I(ins, attr, reg1, reg2, 0, opt); + return; -/***************************************************************************** - * - * Add a label instruction. - */ + case INS_staddb: + emitIns_R_R_R(INS_ldaddb, attr, reg1, REG_ZR, reg2); + return; + case INS_staddlb: + emitIns_R_R_R(INS_ldaddlb, attr, reg1, REG_ZR, reg2); + return; + case INS_staddh: + emitIns_R_R_R(INS_ldaddh, attr, reg1, REG_ZR, reg2); + return; + case INS_staddlh: + emitIns_R_R_R(INS_ldaddlh, attr, reg1, REG_ZR, reg2); + return; + case INS_stadd: + emitIns_R_R_R(INS_ldadd, attr, reg1, REG_ZR, reg2); + return; + case INS_staddl: + emitIns_R_R_R(INS_ldaddl, attr, reg1, REG_ZR, reg2); + return; -void emitter::emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg) -{ - assert(dst->HasFlag(BBF_HAS_LABEL)); + case INS_fcmp: + case INS_fcmpe: + assert(insOptsNone(opt)); + assert(isValidVectorElemsizeFloat(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_DV_2K; + break; - insFormat fmt = IF_NONE; + case INS_fcvtns: + case INS_fcvtnu: + case INS_fcvtas: + case INS_fcvtau: + case INS_fcvtps: + case INS_fcvtpu: + case INS_fcvtms: + case INS_fcvtmu: + case INS_fcvtzs: + case INS_fcvtzu: + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsizeFloat(elemsize)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_2A; + } + else + { + // Scalar operation + assert(isVectorRegister(reg2)); + if (isVectorRegister(reg1)) + { + assert(insOptsNone(opt)); + assert(isValidVectorElemsizeFloat(size)); + fmt = IF_DV_2G; + } + else + { + assert(isGeneralRegister(reg1)); + assert(insOptsConvertFloatToInt(opt)); + assert(isValidVectorElemsizeFloat(size)); + fmt = IF_DV_2H; + } + } + break; - switch (ins) - { - case INS_adr: - fmt = IF_LARGEADR; + case INS_fcvtl: + case INS_fcvtn: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(size == EA_8BYTE); + assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); + fmt = IF_DV_2A; break; - default: - unreached(); - } - instrDescJmp* id = emitNewInstrJmp(); + case INS_fcvtl2: + case INS_fcvtn2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(size == EA_16BYTE); + assert((opt == INS_OPTS_8H) || (opt == INS_OPTS_4S)); + fmt = IF_DV_2A; + break; - id->idIns(ins); - id->idInsFmt(fmt); - id->idjShort = false; - id->idAddr()->iiaBBlabel = dst; - id->idReg1(reg); - id->idOpSize(EA_PTRSIZE); + case INS_fcvtxn: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); -#ifdef DEBUG - // Mark the catch return - if (emitComp->compCurBB->KindIs(BBJ_EHCATCHRET)) - { - id->idDebugOnlyInfo()->idCatchRet = true; - } -#endif // DEBUG + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(size == EA_8BYTE); + assert(opt == INS_OPTS_2S); + fmt = IF_DV_2A; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(size == EA_4BYTE); + fmt = IF_DV_2G; + } + break; - id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); + case INS_fcvtxn2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(size == EA_16BYTE); + assert(opt == INS_OPTS_4S); + fmt = IF_DV_2A; + break; -#ifdef DEBUG - if (emitComp->opts.compLongAddress) - id->idjKeepLong = 1; -#endif // DEBUG + case INS_scvtf: + case INS_ucvtf: + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsizeFloat(elemsize)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_2A; + } + else + { + // Scalar operation + assert(isVectorRegister(reg1)); + if (isVectorRegister(reg2)) + { + assert(insOptsNone(opt)); + assert(isValidVectorElemsizeFloat(size)); + fmt = IF_DV_2G; + } + else + { + assert(isGeneralRegister(reg2)); + assert(insOptsConvertIntToFloat(opt)); + assert(isValidVectorElemsizeFloat(size)); + fmt = IF_DV_2I; + } + } + break; - /* Record the jump's IG and offset within it */ + case INS_fabs: + case INS_fneg: + case INS_fsqrt: + case INS_frinta: + case INS_frinti: + case INS_frintm: + case INS_frintn: + case INS_frintp: + case INS_frintx: + case INS_frintz: + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsizeFloat(elemsize)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_2A; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(isValidVectorElemsizeFloat(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_DV_2G; + } + break; - id->idjIG = emitCurIG; - id->idjOffs = emitCurIGsize; + case INS_faddp: + case INS_fmaxnmp: + case INS_fmaxp: + case INS_fminnmp: + case INS_fminp: + // Scalar operation + assert(((size == EA_8BYTE) && (opt == INS_OPTS_2S)) || ((size == EA_16BYTE) && (opt == INS_OPTS_2D))); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_DV_2Q; + break; - /* Append this jump to this IG's jump list */ + case INS_fmaxnmv: + case INS_fmaxv: + case INS_fminnmv: + case INS_fminv: + assert(size == EA_16BYTE); + assert(opt == INS_OPTS_4S); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_DV_2R; + break; - id->idjNext = emitCurIGjmpList; - emitCurIGjmpList = id; + case INS_addp: + assert(size == EA_16BYTE); + assert(opt == INS_OPTS_2D); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_DV_2S; + break; -#if EMITTER_STATS - emitTotalIGjmps++; -#endif + case INS_fcvt: + assert(insOptsConvertFloatToFloat(opt)); + assert(isValidVectorFcvtsize(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_DV_2J; + break; - dispIns(id); - appendToCurIG(id); -} + case INS_cmeq: + case INS_cmge: + case INS_cmgt: + case INS_cmle: + case INS_cmlt: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); -/***************************************************************************** - * - * Add a data label instruction. - */ + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + fmt = IF_DV_2M; + } + else + { + // Scalar operation + assert(size == EA_8BYTE); + assert(insOptsNone(opt)); + fmt = IF_DV_2L; + } + break; -void emitter::emitIns_R_D(instruction ins, emitAttr attr, unsigned offs, regNumber reg) -{ - NYI("emitIns_R_D"); -} + case INS_fcmeq: + case INS_fcmge: + case INS_fcmgt: + case INS_fcmle: + case INS_fcmlt: + case INS_frecpe: + case INS_frsqrte: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); -void emitter::emitIns_J_R(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg) -{ - assert((ins == INS_cbz) || (ins == INS_cbnz)); + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsizeFloat(elemsize)); // Only Double/Float supported + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_2A; + } + else + { + // Scalar operation + assert(isValidScalarDatasize(size)); // Only Double/Float supported + assert(insOptsNone(opt)); + fmt = IF_DV_2G; + } + break; - assert(dst != nullptr); - assert(dst->HasFlag(BBF_HAS_LABEL)); + case INS_aesd: + case INS_aese: + case INS_aesmc: + case INS_aesimc: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + elemsize = optGetElemsize(opt); + assert(elemsize == EA_1BYTE); + fmt = IF_DV_2P; + break; - insFormat fmt = IF_LARGEJMP; + case INS_sha1h: + assert(insOptsNone(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_DV_2U; + break; - instrDescJmp* id = emitNewInstrJmp(); + case INS_sha256su0: + case INS_sha1su1: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + elemsize = optGetElemsize(opt); + assert(elemsize == EA_4BYTE); + fmt = IF_DV_2P; + break; - id->idIns(ins); - id->idInsFmt(fmt); - id->idReg1(reg); - id->idjShort = false; - id->idOpSize(EA_SIZE(attr)); + case INS_ld2: + case INS_ld3: + case INS_ld4: + case INS_st2: + case INS_st3: + case INS_st4: + assert(opt != INS_OPTS_1D); // .1D format only permitted with LD1 & ST1 + FALLTHROUGH; - id->idAddr()->iiaBBlabel = dst; - id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); + case INS_ld1: + case INS_ld1_2regs: + case INS_ld1_3regs: + case INS_ld1_4regs: + case INS_st1: + case INS_st1_2regs: + case INS_st1_3regs: + case INS_st1_4regs: + case INS_ld1r: + case INS_ld2r: + case INS_ld3r: + case INS_ld4r: + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrSP(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); - /* Record the jump's IG and offset within it */ + // Load/Store multiple structures base register + // Load single structure and replicate base register + reg2 = encodingSPtoZR(reg2); + fmt = IF_LS_2D; + break; - id->idjIG = emitCurIG; - id->idjOffs = emitCurIGsize; + case INS_urecpe: + case INS_ursqrte: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(elemsize == EA_4BYTE); + fmt = IF_DV_2A; + break; - /* Append this jump to this IG's jump list */ + case INS_frecpx: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidScalarDatasize(size)); + assert(insOptsNone(opt)); + fmt = IF_DV_2G; + break; - id->idjNext = emitCurIGjmpList; - emitCurIGjmpList = id; + case INS_sadalp: + case INS_saddlp: + case INS_uadalp: + case INS_uaddlp: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidArrangement(size, opt)); + assert((opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // The encoding size = 11, Q = x is reserved + fmt = IF_DV_2T; + break; -#if EMITTER_STATS - emitTotalIGjmps++; -#endif + case INS_sqabs: + case INS_sqneg: + case INS_suqadd: + case INS_usqadd: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); - dispIns(id); - appendToCurIG(id); -} + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = 0 is reserved + fmt = IF_DV_2M; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(isValidVectorElemsize(size)); + fmt = IF_DV_2L; + } + break; -void emitter::emitIns_J_R_I(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg, int imm) -{ - assert((ins == INS_tbz) || (ins == INS_tbnz)); + default: + // fallback to emit SVE instructions. + return emitInsSve_R_R(ins, attr, reg1, reg2, opt, sopt); - assert(dst != nullptr); - assert(dst->HasFlag(BBF_HAS_LABEL)); - assert((EA_SIZE(attr) == EA_4BYTE) || (EA_SIZE(attr) == EA_8BYTE)); - assert(imm < ((EA_SIZE(attr) == EA_4BYTE) ? 32 : 64)); + } // end switch (ins) - insFormat fmt = IF_LARGEJMP; + assert(fmt != IF_NONE); - instrDescJmp* id = emitNewInstrJmp(); + instrDesc* id = emitNewInstrSmall(attr); id->idIns(ins); id->idInsFmt(fmt); - id->idReg1(reg); - id->idjShort = false; - id->idSmallCns(imm); - id->idOpSize(EA_SIZE(attr)); - - id->idAddr()->iiaBBlabel = dst; - id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); - - /* Record the jump's IG and offset within it */ - - id->idjIG = emitCurIG; - id->idjOffs = emitCurIGsize; - - /* Append this jump to this IG's jump list */ - - id->idjNext = emitCurIGjmpList; - emitCurIGjmpList = id; + id->idInsOpt(opt); -#if EMITTER_STATS - emitTotalIGjmps++; -#endif + id->idReg1(reg1); + id->idReg2(reg2); dispIns(id); appendToCurIG(id); } -void emitter::emitIns_J(instruction ins, BasicBlock* dst, int instrCount) -{ - insFormat fmt = IF_NONE; +/***************************************************************************** + * + * Add an instruction referencing a register and two constants. + */ - if (dst != nullptr) - { - assert(dst->HasFlag(BBF_HAS_LABEL)); - } - else - { - assert(instrCount != 0); - } +void emitter::emitIns_R_I_I(instruction ins, + emitAttr attr, + regNumber reg, + ssize_t imm1, + ssize_t imm2, + insOpts opt /* = INS_OPTS_NONE */ + DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = 0 */)) +{ + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + size_t immOut = 0; // composed from imm1 and imm2 and stored in the instrDesc /* Figure out the encoding format of the instruction */ - switch (ins) { - case INS_bl_local: - case INS_b: - // Unconditional jump is a single form. - // Assume is long in case we cross hot/cold sections. - fmt = IF_BI_0A; - break; - - case INS_beq: - case INS_bne: - case INS_bhs: - case INS_blo: - case INS_bmi: - case INS_bpl: - case INS_bvs: - case INS_bvc: - case INS_bhi: - case INS_bls: - case INS_bge: - case INS_blt: - case INS_bgt: - case INS_ble: - // Assume conditional jump is long. - fmt = IF_LARGEJMP; - break; - - default: - unreached(); - break; - } - - instrDescJmp* id = emitNewInstrJmp(); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idjShort = false; - -#ifdef DEBUG - // Mark the finally call - if (ins == INS_bl_local && emitComp->compCurBB->KindIs(BBJ_CALLFINALLY)) - { - id->idDebugOnlyInfo()->idFinallyCall = true; - } -#endif // DEBUG + bool canEncode; + halfwordImm hwi; - if (dst != nullptr) - { - id->idAddr()->iiaBBlabel = dst; + case INS_mov: + ins = INS_movz; // INS_mov with LSL is an alias for INS_movz LSL + FALLTHROUGH; - // Skip unconditional jump that has a single form. - // The target needs to be relocated. - id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); + case INS_movk: + case INS_movn: + case INS_movz: + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg)); + assert(isValidUimm<16>(imm1)); + assert(insOptsLSL(opt)); // Must be INS_OPTS_LSL -#ifdef DEBUG - if (emitComp->opts.compLongAddress) // Force long branches - { - id->idjKeepLong = true; - } -#endif // DEBUG - } - else - { - id->idAddr()->iiaSetInstrCount(instrCount); - id->idjKeepLong = false; - /* This jump must be short */ - emitSetShortJump(id); - id->idSetIsBound(); - } + if (size == EA_8BYTE) + { + assert((imm2 == 0) || (imm2 == 16) || // shift amount: 0, 16, 32 or 48 + (imm2 == 32) || (imm2 == 48)); + } + else // EA_4BYTE + { + assert((imm2 == 0) || (imm2 == 16)); // shift amount: 0 or 16 + } - /* Record the jump's IG and offset within it */ + hwi.immHWVal = 0; - id->idjIG = emitCurIG; - id->idjOffs = emitCurIGsize; + switch (imm2) + { + case 0: + hwi.immHW = 0; + canEncode = true; + break; - /* Append this jump to this IG's jump list */ + case 16: + hwi.immHW = 1; + canEncode = true; + break; - id->idjNext = emitCurIGjmpList; - emitCurIGjmpList = id; + case 32: + hwi.immHW = 2; + canEncode = true; + break; -#if EMITTER_STATS - emitTotalIGjmps++; -#endif + case 48: + hwi.immHW = 3; + canEncode = true; + break; - dispIns(id); - appendToCurIG(id); -} + default: + canEncode = false; + } -/***************************************************************************** - * - * Add a call instruction (direct or indirect). - * argSize<0 means that the caller will pop the arguments - * - * The other arguments are interpreted depending on callType as shown: - * Unless otherwise specified, ireg,xreg,xmul,disp should have default values. - * - * EC_FUNC_TOKEN : addr is the method address - * EC_FUNC_ADDR : addr is the absolute address of the function - * - * If callType is one of these emitCallTypes, addr has to be NULL. - * EC_INDIR_R : "call ireg". - * - * For ARM xreg, xmul and disp are never used and should always be 0/REG_NA. - * - * Please consult the "debugger team notification" comment in genFnProlog(). - */ + if (canEncode) + { + hwi.immVal = imm1; -void emitter::emitIns_Call(EmitCallType callType, - CORINFO_METHOD_HANDLE methHnd, - INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) // used to report call sites to the EE - void* addr, - ssize_t argSize, - emitAttr retSize, - emitAttr secondRetSize, - VARSET_VALARG_TP ptrVars, - regMaskTP gcrefRegs, - regMaskTP byrefRegs, - const DebugInfo& di /* = DebugInfo() */, - regNumber ireg /* = REG_NA */, - regNumber xreg /* = REG_NA */, - unsigned xmul /* = 0 */, - ssize_t disp /* = 0 */, - bool isJump /* = false */) -{ - /* Sanity check the arguments depending on callType */ + immOut = hwi.immHWVal; + assert(isValidImmHWVal(immOut, size)); + fmt = IF_DI_1B; + } + break; - assert(callType < EC_COUNT); - assert((callType != EC_FUNC_TOKEN) || (addr != nullptr && ireg == REG_NA)); - assert(callType != EC_INDIR_R || (addr == nullptr && ireg < REG_COUNT)); + default: + // fallback to emit SVE instructions. + return emitInsSve_R_I_I(ins, attr, reg, imm1, imm2, opt); - // ARM never uses these - assert(xreg == REG_NA && xmul == 0 && disp == 0); + } // end switch (ins) - // Our stack level should be always greater than the bytes of arguments we push. Just - // a sanity test. - assert((unsigned)abs(argSize) <= codeGen->genStackLevel); + assert(fmt != IF_NONE); - // Trim out any callee-trashed registers from the live set. - regMaskTP savedSet = emitGetGCRegsSavedOrModified(methHnd); - gcrefRegs &= savedSet; - byrefRegs &= savedSet; + instrDesc* id = emitNewInstrSC(attr, immOut); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg); #ifdef DEBUG - if (EMIT_GC_VERBOSE) - { - printf("Call: GCvars=%s ", VarSetOps::ToString(emitComp, ptrVars)); - dumpConvertedVarSet(emitComp, ptrVars); - printf(", gcrefRegs="); - printRegMaskInt(gcrefRegs); - emitDispRegSet(gcrefRegs); - printf(", byrefRegs="); - printRegMaskInt(byrefRegs); - emitDispRegSet(byrefRegs); - printf("\n"); - } + id->idDebugOnlyInfo()->idFlags = gtFlags; + id->idDebugOnlyInfo()->idMemCookie = targetHandle; #endif - /* Managed RetVal: emit sequence point for the call */ - if (emitComp->opts.compDbgInfo && di.GetLocation().IsValid()) - { - codeGen->genIPmappingAdd(IPmappingDscKind::Normal, di, false); - } + dispIns(id); + appendToCurIG(id); +} - /* - We need to allocate the appropriate instruction descriptor based - on whether this is a direct/indirect call, and whether we need to - record an updated set of live GC variables. - */ - instrDesc* id; +/***************************************************************************** + * + * Add an instruction referencing two registers and a constant. + */ - assert(argSize % REGSIZE_BYTES == 0); - int argCnt = (int)(argSize / (int)REGSIZE_BYTES); +void emitter::emitIns_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +{ + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; + bool isLdSt = false; + bool isLdrStr = false; + bool isSIMD = false; + bool isAddSub = false; + bool setFlags = false; + unsigned scale = 0; + bool unscaledOp = false; - if (callType == EC_INDIR_R) + /* Figure out the encoding format of the instruction */ + switch (ins) { - /* Indirect call, virtual calls */ + bool canEncode; + bitMaskImm bmi; + unsigned registerListSize; + bool isRightShift; - id = emitNewInstrCallInd(argCnt, 0 /* disp */, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize); - } - else - { - /* Helper/static/nonvirtual/function calls (direct or through handle), - and calls to an absolute addr. */ + case INS_mov: + // Check for the 'mov' aliases for the vector registers + assert(insOptsNone(opt)); + assert(isValidVectorElemsize(size)); + elemsize = size; + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - assert(callType == EC_FUNC_TOKEN); + if (isVectorRegister(reg1)) + { + if (isGeneralRegisterOrZR(reg2)) + { + fmt = IF_DV_2C; // Alias for 'ins' + break; + } + else if (isVectorRegister(reg2)) + { + fmt = IF_DV_2E; // Alias for 'dup' + break; + } + } + else // isGeneralRegister(reg1) + { + assert(isGeneralRegister(reg1)); + if (isVectorRegister(reg2)) + { + fmt = IF_DV_2B; // Alias for 'umov' + break; + } + } + assert(!" invalid INS_mov operands"); + break; - id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize); - } + case INS_lsl: + case INS_lsr: + case INS_asr: + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + assert(isValidImmShift(imm, size)); + fmt = IF_DI_2D; + break; - /* Update the emitter's live GC ref sets */ + case INS_ror: + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + assert(isValidImmShift(imm, size)); + fmt = IF_DI_2B; + break; - VarSetOps::Assign(emitComp, emitThisGCrefVars, ptrVars); - emitThisGCrefRegs = gcrefRegs; - emitThisByrefRegs = byrefRegs; + case INS_shl: + case INS_sli: + case INS_sri: + case INS_srshr: + case INS_srsra: + case INS_sshr: + case INS_ssra: + case INS_urshr: + case INS_ursra: + case INS_ushr: + case INS_usra: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + isRightShift = emitInsIsVectorRightShift(ins); - id->idSetIsNoGC(emitNoGChelper(methHnd)); + assert(!isRightShift || + (imm != 0 && "instructions for vector right-shift do not allow zero as an immediate value")); + + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsize(elemsize)); + assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_2O; + break; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(size == EA_8BYTE); // only supported size + assert(isValidVectorShiftAmount(imm, size, isRightShift)); + fmt = IF_DV_2N; + } + break; + + case INS_sqshl: + case INS_uqshl: + case INS_sqshlu: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + isRightShift = emitInsIsVectorRightShift(ins); + + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_1D); // The encoding immh = 1xxx, Q = 0 is reserved + elemsize = optGetElemsize(opt); + assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); + fmt = IF_DV_2O; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(isValidVectorElemsize(size)); + assert(isValidVectorShiftAmount(imm, size, isRightShift)); + fmt = IF_DV_2N; + } + break; + + case INS_sqrshrn: + case INS_sqrshrun: + case INS_sqshrn: + case INS_sqshrun: + case INS_uqrshrn: + case INS_uqshrn: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + isRightShift = emitInsIsVectorRightShift(ins); + + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidArrangement(size, opt)); + assert((opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // The encoding immh = 1xxx, Q = x is reserved + elemsize = optGetElemsize(opt); + assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); + fmt = IF_DV_2O; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(isValidVectorElemsize(size)); + assert(size != EA_8BYTE); // The encoding immh = 1xxx is reserved + assert(isValidVectorShiftAmount(imm, size, isRightShift)); + fmt = IF_DV_2N; + } + break; + + case INS_sxtl: + case INS_uxtl: + assert(imm == 0); + FALLTHROUGH; + + case INS_rshrn: + case INS_shrn: + case INS_sshll: + case INS_ushll: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + isRightShift = emitInsIsVectorRightShift(ins); + // Vector operation + assert(size == EA_8BYTE); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(elemsize != EA_8BYTE); // Reserved encodings + assert(isValidVectorElemsize(elemsize)); + assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); + fmt = IF_DV_2O; + break; - /* Set the instruction - special case jumping a function */ - instruction ins; - insFormat fmt = IF_NONE; + case INS_sxtl2: + case INS_uxtl2: + assert(imm == 0); + FALLTHROUGH; - /* Record the address: method, indirection, or funcptr */ + case INS_rshrn2: + case INS_shrn2: + case INS_sqrshrn2: + case INS_sqrshrun2: + case INS_sqshrn2: + case INS_sqshrun2: + case INS_sshll2: + case INS_uqrshrn2: + case INS_uqshrn2: + case INS_ushll2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + isRightShift = emitInsIsVectorRightShift(ins); - if (callType == EC_INDIR_R) - { - /* This is an indirect call (either a virtual call or func ptr call) */ + // Vector operation + assert(size == EA_16BYTE); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(elemsize != EA_8BYTE); // The encoding immh = 1xxx, Q = x is reserved + assert(isValidVectorElemsize(elemsize)); + assert(isValidVectorShiftAmount(imm, elemsize, isRightShift)); + fmt = IF_DV_2O; + break; - if (isJump) - { - ins = INS_br_tail; // INS_br_tail Reg - } - else - { - ins = INS_blr; // INS_blr Reg - } - fmt = IF_BR_1B; + case INS_mvn: + case INS_neg: + case INS_negs: + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); - id->idIns(ins); - id->idInsFmt(fmt); + if (imm == 0) + { + assert(insOptsNone(opt)); // a zero imm, means no alu shift kind - assert(xreg == REG_NA); - if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && EA_IS_CNS_TLSGD_RELOC(retSize)) - { - // For NativeAOT linux/arm64, we need to also record the relocation of methHnd. - // Since we do not have space to embed it in instrDesc, we store the register in - // reg1 and instead use the `iiaAdd` to store the method handle. Likewise, during - // emitOutputInstr, we retrieve the register from reg1 for this specific case. - id->idSetTlsGD(); - id->idReg1(ireg); - id->idAddr()->iiaAddr = (BYTE*)methHnd; - } - else - { - id->idReg3(ireg); - } - } - else - { - /* This is a simple direct call: "call helper/method/addr" */ + fmt = IF_DR_2E; + } + else + { + if (ins == INS_mvn) + { + assert(insOptsAnyShift(opt)); // a non-zero imm, must select shift kind + } + else // neg or negs + { + assert(insOptsAluShift(opt)); // a non-zero imm, must select shift kind, can't use ROR + } + assert(isValidImmShift(imm, size)); + fmt = IF_DR_2F; + } + break; - assert(callType == EC_FUNC_TOKEN); + case INS_tst: + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegister(reg2)); - assert(addr != NULL); + if (insOptsAnyShift(opt)) + { + assert(isValidImmShift(imm, size) && (imm != 0)); + fmt = IF_DR_2B; + } + else + { + assert(insOptsNone(opt)); // a zero imm, means no alu shift kind + assert(imm == 0); + fmt = IF_DR_2A; + } + break; - if (isJump) - { - ins = INS_b_tail; // INS_b_tail imm28 - } - else - { - ins = INS_bl; // INS_bl imm28 - } - fmt = IF_BI_0C; + case INS_cmp: + case INS_cmn: + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegisterOrSP(reg1)); + assert(isGeneralRegister(reg2)); - id->idIns(ins); - id->idInsFmt(fmt); + reg1 = encodingSPtoZR(reg1); + if (insOptsAnyExtend(opt)) + { + assert((imm >= 0) && (imm <= 4)); - id->idAddr()->iiaAddr = (BYTE*)addr; + fmt = IF_DR_2C; + } + else if (imm == 0) + { + assert(insOptsNone(opt)); // a zero imm, means no alu shift kind - if (emitComp->opts.compReloc) - { - id->idSetIsDspReloc(); - } - } + fmt = IF_DR_2A; + } + else + { + assert(insOptsAnyShift(opt)); // a non-zero imm, must select shift kind + assert(isValidImmShift(imm, size)); + fmt = IF_DR_2B; + } + break; -#ifdef DEBUG - if (EMIT_GC_VERBOSE) - { - if (id->idIsLargeCall()) - { - printf("[%02u] Rec call GC vars = %s\n", id->idDebugOnlyInfo()->idNum, - VarSetOps::ToString(emitComp, ((instrDescCGCA*)id)->idcGCvars)); - } - } -#endif + case INS_ands: + case INS_and: + case INS_eor: + case INS_orr: + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg2)); + if (ins == INS_ands) + { + assert(isGeneralRegister(reg1)); + } + else + { + assert(isGeneralRegisterOrSP(reg1)); + reg1 = encodingSPtoZR(reg1); + } - if (m_debugInfoSize > 0) - { - INDEBUG(id->idDebugOnlyInfo()->idCallSig = sigInfo); - id->idDebugOnlyInfo()->idMemCookie = (size_t)methHnd; // method token - } + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, size, &bmi); + if (canEncode) + { + imm = bmi.immNRS; + assert(isValidImmNRS(imm, size)); + fmt = IF_DI_2C; + } + break; -#ifdef LATE_DISASM - if (addr != nullptr) - { - codeGen->getDisAssembler().disSetMethod((size_t)addr, methHnd); - } -#endif // LATE_DISASM + case INS_dup: // by element, imm selects the element of reg2 + assert(isVectorRegister(reg1)); + if (isVectorRegister(reg2)) + { + if (insOptsAnyArrangement(opt)) + { + // The size and opt were modified to be based on the + // return type but the immediate is based on the operand + // which can be of a larger size. As such, we don't + // assert the index is valid here and instead do it in + // codegen. - dispIns(id); - appendToCurIG(id); - emitLastMemBarrier = nullptr; // Cannot optimize away future memory barriers -} + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsize(elemsize)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_2D; + break; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + elemsize = size; + assert(isValidVectorElemsize(elemsize)); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + fmt = IF_DV_2E; + break; + } + } + FALLTHROUGH; -/***************************************************************************** - * - * Returns true if 'imm' is valid Cond encoding - */ + case INS_ins: // (MOV from general) + assert(insOptsNone(opt)); + assert(isValidVectorElemsize(size)); + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + elemsize = size; + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + fmt = IF_DV_2C; + break; -/*static*/ bool emitter::isValidImmCond(ssize_t imm) -{ - // range check the ssize_t value, to make sure it is a small unsigned value - // and that only the bits in the cfi.cond are set - if ((imm < 0) || (imm > 0xF)) - return false; + case INS_umov: // (MOV to general) + assert(insOptsNone(opt)); + assert(isValidVectorElemsize(size)); + assert(isGeneralRegister(reg1)); + assert(isVectorRegister(reg2)); + elemsize = size; + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + fmt = IF_DV_2B; + break; - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; + case INS_smov: + assert(insOptsNone(opt)); + assert(isValidVectorElemsize(size)); + assert(size != EA_8BYTE); // no encoding, use INS_umov + assert(isGeneralRegister(reg1)); + assert(isVectorRegister(reg2)); + elemsize = size; + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + fmt = IF_DV_2B; + break; - return (cfi.cond <= INS_COND_LE); // Don't allow 14 & 15 (AL & NV). -} + case INS_add: + case INS_sub: + setFlags = false; + isAddSub = true; + break; -/***************************************************************************** - * - * Returns true if 'imm' is valid Cond/Flags encoding - */ + case INS_adds: + case INS_subs: + setFlags = true; + isAddSub = true; + break; -/*static*/ bool emitter::isValidImmCondFlags(ssize_t imm) -{ - // range check the ssize_t value, to make sure it is a small unsigned value - // and that only the bits in the cfi.cond or cfi.flags are set - if ((imm < 0) || (imm > 0xFF)) - return false; + case INS_ldrsb: + case INS_ldursb: + // 'size' specifies how we sign-extend into 4 or 8 bytes of the target register + assert(isValidGeneralDatasize(size)); + unscaledOp = (ins == INS_ldursb); + scale = 0; + isLdSt = true; + break; - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; + case INS_ldrsh: + case INS_ldursh: + // 'size' specifies how we sign-extend into 4 or 8 bytes of the target register + assert(isValidGeneralDatasize(size)); + unscaledOp = (ins == INS_ldursh); + scale = 1; + isLdSt = true; + break; - return (cfi.cond <= INS_COND_LE); // Don't allow 14 & 15 (AL & NV). -} + case INS_ldrsw: + case INS_ldursw: + // 'size' specifies how we sign-extend into 4 or 8 bytes of the target register + assert(size == EA_8BYTE); + unscaledOp = (ins == INS_ldursw); + scale = 2; + isLdSt = true; + break; -/***************************************************************************** - * - * Returns true if 'imm' is valid Cond/Flags/Imm5 encoding - */ + case INS_ldrb: + case INS_strb: + // size is ignored + unscaledOp = false; + scale = 0; + isLdSt = true; + break; -/*static*/ bool emitter::isValidImmCondFlagsImm5(ssize_t imm) -{ - // range check the ssize_t value, to make sure it is a small unsigned value - // and that only the bits in the cfi.cond, cfi.flags or cfi.imm5 are set - if ((imm < 0) || (imm > 0x1FFF)) - return false; + case INS_ldapurb: + case INS_stlurb: + case INS_ldurb: + case INS_sturb: + // size is ignored + unscaledOp = true; + scale = 0; + isLdSt = true; + break; - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; + case INS_ldrh: + case INS_strh: + // size is ignored + unscaledOp = false; + scale = 1; + isLdSt = true; + break; - return (cfi.cond <= INS_COND_LE); // Don't allow 14 & 15 (AL & NV). -} + case INS_ldurh: + case INS_ldapurh: + case INS_sturh: + case INS_stlurh: + // size is ignored + unscaledOp = true; + scale = 0; + isLdSt = true; + break; -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Rd' position - */ + case INS_ldr: + case INS_str: + // Is the target a vector register? + if (isVectorRegister(reg1)) + { + assert(isValidVectorLSDatasize(size)); + assert(isGeneralRegisterOrSP(reg2)); + isSIMD = true; + } + else + { + assert(isValidGeneralDatasize(size)); + } + unscaledOp = false; + scale = NaturalScale_helper(size); + isLdSt = true; + isLdrStr = true; + break; -/*static*/ emitter::code_t emitter::insEncodeReg_Rd(regNumber reg) -{ - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 0) && (ureg <= 31)); - return ureg; -} + case INS_ldur: + case INS_stur: + case INS_ldapur: + case INS_stlur: + // Is the target a vector register? + if (isVectorRegister(reg1)) + { + assert(isValidVectorLSDatasize(size)); + assert(isGeneralRegisterOrSP(reg2)); + isSIMD = true; + } + else + { + assert(isValidGeneralDatasize(size)); + } + unscaledOp = true; + scale = 0; + isLdSt = true; + break; -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Rt' position - */ + case INS_ld2: + case INS_ld3: + case INS_ld4: + case INS_st2: + case INS_st3: + case INS_st4: + assert(opt != INS_OPTS_1D); // .1D format only permitted with LD1 & ST1 + FALLTHROUGH; -/*static*/ emitter::code_t emitter::insEncodeReg_Rt(regNumber reg) -{ - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 0) && (ureg <= 31)); - return ureg; -} + case INS_ld1: + case INS_ld1_2regs: + case INS_ld1_3regs: + case INS_ld1_4regs: + case INS_st1: + case INS_st1_2regs: + case INS_st1_3regs: + case INS_st1_4regs: + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrSP(reg2)); -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Rn' position - */ + reg2 = encodingSPtoZR(reg2); -/*static*/ emitter::code_t emitter::insEncodeReg_Rn(regNumber reg) -{ - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 5; -} + if (insOptsAnyArrangement(opt)) + { + registerListSize = insGetRegisterListSize(ins); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + assert((size * registerListSize) == imm); -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Rm' position - */ + // Load/Store multiple structures post-indexed by an immediate + fmt = IF_LS_2E; + } + else + { + assert(insOptsNone(opt)); + assert((ins != INS_ld1_2regs) && (ins != INS_ld1_3regs) && (ins != INS_ld1_4regs) && + (ins != INS_st1_2regs) && (ins != INS_st1_3regs) && (ins != INS_st1_4regs)); -/*static*/ emitter::code_t emitter::insEncodeReg_Rm(regNumber reg) -{ - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 16; -} + elemsize = size; + assert(isValidVectorElemsize(elemsize)); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Ra' position - */ + // Load/Store single structure base register + fmt = IF_LS_2F; + } + break; -/*static*/ emitter::code_t emitter::insEncodeReg_Ra(regNumber reg) -{ - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 10; -} + case INS_ld1r: + case INS_ld2r: + case INS_ld3r: + case INS_ld4r: + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrSP(reg2)); -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Vd' position - */ + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); -/*static*/ emitter::code_t emitter::insEncodeReg_Vd(regNumber reg) -{ - assert(emitter::isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 31)); - return ureg; -} + elemsize = optGetElemsize(opt); + registerListSize = insGetRegisterListSize(ins); + assert((elemsize * registerListSize) == imm); -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Vt' position - */ + // Load single structure and replicate post-indexed by an immediate + reg2 = encodingSPtoZR(reg2); + fmt = IF_LS_2E; + break; -/*static*/ emitter::code_t emitter::insEncodeReg_Vt(regNumber reg) -{ - assert(emitter::isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 31)); - return ureg; -} + default: + // fallback to emit SVE instructions. + return emitInsSve_R_R_I(ins, attr, reg1, reg2, imm, opt, sopt); -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Vn' position - */ + } // end switch (ins) -/*static*/ emitter::code_t emitter::insEncodeReg_Vn(regNumber reg) -{ - assert(emitter::isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 5; -} + if (isLdSt) + { + assert(!isAddSub); -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Vm' position - */ + if (isSIMD) + { + assert(isValidVectorLSDatasize(size)); + assert(isVectorRegister(reg1)); + assert((scale >= 0) && (scale <= 4)); + } + else + { + assert(isValidGeneralLSDatasize(size)); + assert(isGeneralRegisterOrZR(reg1)); + assert((scale >= 0) && (scale <= 3)); + } -/*static*/ emitter::code_t emitter::insEncodeReg_Vm(regNumber reg) -{ - assert(emitter::isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 16; -} + assert(isGeneralRegisterOrSP(reg2)); -/***************************************************************************** - * - * Returns an encoding for the specified register used in the 'Va' position - */ + // Load/Store reserved encodings: + if (insOptsIndexed(opt)) + { + assert(reg1 != reg2); + } -/*static*/ emitter::code_t emitter::insEncodeReg_Va(regNumber reg) -{ - assert(emitter::isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 10; -} + reg2 = encodingSPtoZR(reg2); -/***************************************************************************** - * - * Return an encoding for the specified 'V' register used in '4' thru '0' position. - */ + ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate + if (imm == 0 || EA_IS_CNS_TLSGD_RELOC(attr)) + { + assert(insOptsNone(opt)); // PRE/POST Index doesn't make sense with an immediate of zero -/*static*/ emitter::code_t emitter::insEncodeReg_V_4_to_0(regNumber reg) -{ - assert(isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 0; -} + fmt = IF_LS_2A; + } + else if (insOptsIndexed(opt) || unscaledOp || (imm < 0) || ((imm & mask) != 0)) + { + if ((imm >= -256) && (imm <= 255)) + { + fmt = IF_LS_2C; + } + else + { + assert(!"Instruction cannot be encoded: IF_LS_2C"); + } + } + else if (imm > 0) + { + assert(insOptsNone(opt)); + assert(!unscaledOp); -/***************************************************************************** - * - * Return an encoding for the specified 'V' register used in '9' thru '5' position. - */ + if (((imm & mask) == 0) && ((imm >> scale) < 0x1000)) + { + imm >>= scale; // The immediate is scaled by the size of the ld/st -/*static*/ emitter::code_t emitter::insEncodeReg_V_9_to_5(regNumber reg) -{ - assert(isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 5; -} + fmt = IF_LS_2B; + } + else + { + assert(!"Instruction cannot be encoded: IF_LS_2B"); + } + } -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '12' thru '10' position. - */ + // Try to optimize a load/store with an alternative instruction. + if (isLdrStr && emitComp->opts.OptimizationEnabled() && + OptimizeLdrStr(ins, attr, reg1, reg2, imm, size, fmt, false, -1, -1 DEBUG_ARG(false))) + { + return; + } + } + else if (isAddSub) + { + assert(!isLdSt); + assert(insOptsNone(opt)); -/*static*/ emitter::code_t emitter::insEncodeReg_P_12_to_10(regNumber reg) -{ - assert(isLowPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 7)); - return ureg << 10; -} + if (setFlags) // Can't encode SP with setFlags + { + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + } + else + { + assert(isGeneralRegisterOrSP(reg1)); + assert(isGeneralRegisterOrSP(reg2)); -/***************************************************************************** - * - * Return an encoding for the specified 'V' register used in '20' thru '16' position. - */ + // Is it just a mov? + if (imm == 0) + { + emitIns_Mov(INS_mov, attr, reg1, reg2, /* canSkip */ true); + return; + } -/*static*/ emitter::code_t emitter::insEncodeReg_V_20_to_16(regNumber reg) -{ - assert(isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 16; -} + reg1 = encodingSPtoZR(reg1); + reg2 = encodingSPtoZR(reg2); + } -/***************************************************************************** - * - * Return an encoding for the specified 'R' register used in '20' thru '16' position. - */ + if (unsigned_abs(imm) <= 0x0fff) + { + if (imm < 0) + { + ins = insReverse(ins); + imm = -imm; + } + assert(isValidUimm<12>(imm)); + fmt = IF_DI_2A; + } + else if (canEncodeWithShiftImmBy12(imm)) // Try the shifted by 12 encoding + { + // Encoding will use a 12-bit left shift of the immediate + opt = INS_OPTS_LSL12; + if (imm < 0) + { + ins = insReverse(ins); + imm = -imm; + } + assert((imm & 0xfff) == 0); + imm >>= 12; + assert(isValidUimm<12>(imm)); + fmt = IF_DI_2A; + } + else + { + assert(!"Instruction cannot be encoded: IF_DI_2A"); + } + } -/*static*/ emitter::code_t emitter::insEncodeReg_R_20_to_16(regNumber reg) -{ - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 16; -} + assert(fmt != IF_NONE); -/***************************************************************************** - * - * Return an encoding for the specified 'R' register used in '9' thru '5' position. - */ + instrDesc* id = emitNewInstrSC(attr, imm); -/*static*/ emitter::code_t emitter::insEncodeReg_R_9_to_5(regNumber reg) -{ - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 5; -} + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); -/***************************************************************************** - * - * Return an encoding for the specified 'R' register used in '4' thru '0' position. - */ + id->idReg1(reg1); + id->idReg2(reg2); -/*static*/ emitter::code_t emitter::insEncodeReg_R_4_to_0(regNumber reg) -{ - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 0; + if (EA_IS_CNS_TLSGD_RELOC(attr)) + { + assert(imm != 0); + id->idSetTlsGD(); + } + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Return an encoding for the specified 'P' register used in '19' thru '16' position. + * Add an instruction referencing two registers and a floating point constant. */ -/*static*/ emitter::code_t emitter::insEncodeReg_P_19_to_16(regNumber reg) +void emitter::emitIns_R_R_F( + instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, double immDbl, insOpts opt /* = INS_OPTS_NONE */) { - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 16; + // Currently, only SVE instructions use this format. + emitInsSve_R_R_F(ins, attr, reg1, reg2, immDbl, opt); } /***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '3' thru '0' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_3_to_0(regNumber reg) +* +* Add an instruction referencing two registers and a constant. +* Also checks for a large immediate that needs a second instruction +* and will load it in reg1 +* +* - Supports instructions: add, adds, sub, subs, and, ands, eor and orr +* - Requires that reg1 is a general register and not SP or ZR +* - Requires that reg1 != reg2 +*/ +void emitter::emitIns_R_R_Imm(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, ssize_t imm) { - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg; -} + assert(isGeneralRegister(reg1)); + assert(reg1 != reg2); -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '8' thru '5' position. - */ + bool immFits = true; -/*static*/ emitter::code_t emitter::insEncodeReg_P_8_to_5(regNumber reg) -{ - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 5; -} + switch (ins) + { + case INS_add: + case INS_adds: + case INS_sub: + case INS_subs: + immFits = emitter::emitIns_valid_imm_for_add(imm, attr); + break; -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '13' thru '10' position. - */ + case INS_ands: + case INS_and: + case INS_eor: + case INS_orr: + immFits = emitter::emitIns_valid_imm_for_alu(imm, attr); + break; -/*static*/ emitter::code_t emitter::insEncodeReg_P_13_to_10(regNumber reg) -{ - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 10; + default: + assert(!"Unsupported instruction in emitIns_R_R_Imm"); + } + + if (immFits) + { + emitIns_R_R_I(ins, attr, reg1, reg2, imm); + } + else + { + // Load 'imm' into the reg1 register + // then issue: 'ins' reg1, reg2, reg1 + // + codeGen->instGen_Set_Reg_To_Imm(attr, reg1, imm); + emitIns_R_R_R(ins, attr, reg1, reg2, reg1); + } } /***************************************************************************** * - * Return an encoding for the specified 'R' register used in '17' thru '16' position. + * Add an instruction referencing three registers. */ -/*static*/ emitter::code_t emitter::insEncodeReg_R_17_to_16(regNumber reg) +void emitter::emitIns_R_R_R(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) { - assert(isIntegerRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg; - assert((ureg >= 12) && (ureg <= 15)); - return ureg << 16; -} + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '7' thru '5' position. - */ + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_mul: + case INS_smull: + case INS_umull: + if (insOptsAnyArrangement(opt)) + { + // ASIMD instruction + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isValidArrangement(size, opt)); + assert((opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // The encoding size = 11, Q = x is reserved + fmt = IF_DV_3A; + break; + } + // Base instruction + FALLTHROUGH; -/*static*/ emitter::code_t emitter::insEncodeReg_P_7_to_5(regNumber reg) -{ - assert(isHighPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P8; - assert((ureg >= 0) && (ureg <= 7)); - return ureg << 5; -} + case INS_lsl: + case INS_lsr: + case INS_asr: + case INS_ror: + case INS_adc: + case INS_adcs: + case INS_sbc: + case INS_sbcs: + case INS_udiv: + case INS_sdiv: + case INS_mneg: + case INS_smnegl: + case INS_smulh: + case INS_umnegl: + case INS_umulh: + case INS_lslv: + case INS_lsrv: + case INS_asrv: + case INS_rorv: + case INS_crc32b: + case INS_crc32h: + case INS_crc32w: + case INS_crc32x: + case INS_crc32cb: + case INS_crc32ch: + case INS_crc32cw: + case INS_crc32cx: + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + assert(isGeneralRegister(reg3)); + fmt = IF_DR_3A; + break; -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '3' thru '1' position. - */ + case INS_add: + case INS_sub: + if (isVectorRegister(reg1)) + { + // ASIMD instruction + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); -/*static*/ emitter::code_t emitter::insEncodeReg_P_3_to_1(regNumber reg) -{ - assert(isLowPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 1; -} + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(opt != INS_OPTS_1D); // Reserved encoding + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + fmt = IF_DV_3A; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(size == EA_8BYTE); + fmt = IF_DV_3E; + } + break; + } + // Base instruction + FALLTHROUGH; -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '2' thru '0' position. - */ + case INS_adds: + case INS_subs: + emitIns_R_R_R_I(ins, attr, reg1, reg2, reg3, 0, opt); + return; -/*static*/ emitter::code_t emitter::insEncodeReg_P_2_to_0(regNumber reg) -{ - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 8) && (ureg <= 15)); - return (ureg - 8) << 0; -} + case INS_cmeq: + case INS_cmge: + case INS_cmgt: + case INS_cmhi: + case INS_cmhs: + case INS_cmtst: + case INS_srshl: + case INS_sshl: + case INS_urshl: + case INS_ushl: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = 0 is reserved + fmt = IF_DV_3A; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(size == EA_8BYTE); // Only Int64/UInt64 supported + fmt = IF_DV_3E; + } + break; + + case INS_sqadd: + case INS_sqrshl: + case INS_sqshl: + case INS_sqsub: + case INS_uqadd: + case INS_uqrshl: + case INS_uqshl: + case INS_uqsub: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = 0 is reserved + fmt = IF_DV_3A; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(isValidVectorElemsize(size)); + fmt = IF_DV_3E; + } + break; + + case INS_fcmeq: + case INS_fcmge: + case INS_fcmgt: + case INS_frecps: + case INS_frsqrts: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); -/***************************************************************************** - * - * Return an encoding for the specified predicate type used in '16' position. - */ + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert((elemsize == EA_8BYTE) || (elemsize == EA_4BYTE)); // Only Double/Float supported + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_3B; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert((size == EA_8BYTE) || (size == EA_4BYTE)); // Only Double/Float supported + fmt = IF_DV_3D; + } + break; -/*static*/ emitter::code_t emitter::insEncodePredQualifier_16(bool merge) -{ - return merge ? 1 << 16 : 0; -} + case INS_mla: + case INS_mls: + case INS_saba: + case INS_sabd: + case INS_shadd: + case INS_shsub: + case INS_smax: + case INS_smaxp: + case INS_smin: + case INS_sminp: + case INS_srhadd: + case INS_uaba: + case INS_uabd: + case INS_uhadd: + case INS_uhsub: + case INS_umax: + case INS_umaxp: + case INS_umin: + case INS_uminp: + case INS_urhadd: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isValidArrangement(size, opt)); + assert((opt != INS_OPTS_1D) && (opt != INS_OPTS_2D)); // The encoding size = 11, Q = x is reserved + fmt = IF_DV_3A; + break; -/***************************************************************************** - * - * Return an encoding for the specified predicate type used in '4' position. - */ + case INS_addp: + case INS_uzp1: + case INS_uzp2: + case INS_zip1: + case INS_zip2: + case INS_trn1: + case INS_trn2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = 0 is reserved + fmt = IF_DV_3A; + break; -/*static*/ emitter::code_t emitter::insEncodePredQualifier_4(bool merge) -{ - return merge ? 1 << 4 : 0; -} + case INS_mov: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(reg2 == reg3); + assert(isValidVectorDatasize(size)); + // INS_mov is an alias for INS_orr (vector register) + if (opt == INS_OPTS_NONE) + { + elemsize = EA_1BYTE; + opt = optMakeArrangement(size, elemsize); + } + assert(isValidArrangement(size, opt)); + fmt = IF_DV_3C; + break; -/***************************************************************************** - * - * Return an encoding for the specified 'V' register used in '18' thru '16' position. - */ + case INS_and: + case INS_bic: + case INS_eor: + case INS_orr: + case INS_orn: + case INS_tbl: + case INS_tbl_2regs: + case INS_tbl_3regs: + case INS_tbl_4regs: + case INS_tbx: + case INS_tbx_2regs: + case INS_tbx_3regs: + case INS_tbx_4regs: + if (isVectorRegister(reg1)) + { + assert(isValidVectorDatasize(size)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (opt == INS_OPTS_NONE) + { + elemsize = EA_1BYTE; + opt = optMakeArrangement(size, elemsize); + } + assert(isValidArrangement(size, opt)); + fmt = IF_DV_3C; + break; + } + FALLTHROUGH; -/*static*/ emitter::code_t emitter::insEncodeReg_V_18_to_16(regNumber reg) -{ - assert(isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 7)); - return ureg << 16; -} + case INS_ands: + case INS_bics: + case INS_eon: + emitIns_R_R_R_I(ins, attr, reg1, reg2, reg3, 0, INS_OPTS_NONE); + return; -/***************************************************************************** - * - * Return an encoding for the specified 'V' register used in '19' thru '16' position. - */ + case INS_bsl: + case INS_bit: + case INS_bif: + assert(isValidVectorDatasize(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (opt == INS_OPTS_NONE) + { + elemsize = EA_1BYTE; + opt = optMakeArrangement(size, elemsize); + } + assert(isValidArrangement(size, opt)); + fmt = IF_DV_3C; + break; -/*static*/ emitter::code_t emitter::insEncodeReg_V_19_to_16(regNumber reg) -{ - assert(isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 16; -} + case INS_fadd: + case INS_fsub: + case INS_fdiv: + case INS_fmax: + case INS_fmaxnm: + case INS_fmin: + case INS_fminnm: + case INS_fabd: + case INS_fmul: + case INS_fmulx: + case INS_facge: + case INS_facgt: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsizeFloat(elemsize)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_3B; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(isValidScalarDatasize(size)); + fmt = IF_DV_3D; + } + break; -/***************************************************************************** - * - * Return an encoding for the specified 'V' register used in '9' thru '6' position. - */ + case INS_fnmul: + // Scalar operation + assert(insOptsNone(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isValidScalarDatasize(size)); + fmt = IF_DV_3D; + break; -/*static*/ emitter::code_t emitter::insEncodeReg_V_9_to_6(regNumber reg) -{ - assert(isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 6; -} + case INS_faddp: + case INS_fmaxnmp: + case INS_fmaxp: + case INS_fminnmp: + case INS_fminp: -/***************************************************************************** - * - * Return an encoding for the specified 'V' register used in '9' thru '6' position with the times two encoding. - * This encoding requires that the register number be divisible by two. - */ + case INS_fmla: + case INS_fmls: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsAnyArrangement(opt)); // no scalar encoding, use 4-operand 'fmadd' or 'fmsub' -/*static*/ emitter::code_t emitter::insEncodeReg_V_9_to_6_Times_Two(regNumber reg) -{ - assert(isVectorRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; - assert(ureg % 2 == 0); - ureg /= 2u; - assert((ureg >= 0) && (ureg <= 31)); - return ureg << 6; -} + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsizeFloat(elemsize)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_3B; + break; -/***************************************************************************** - * - * Returns an encoding for the specified condition code. - */ + case INS_ldr: + case INS_ldrb: + case INS_ldrh: + case INS_ldrsb: + case INS_ldrsh: + case INS_ldrsw: + case INS_str: + case INS_strb: + case INS_strh: + emitIns_R_R_R_Ext(ins, attr, reg1, reg2, reg3, opt); + return; -/*static*/ emitter::code_t emitter::insEncodeCond(insCond cond) -{ - emitter::code_t uimm = (emitter::code_t)cond; - return uimm << 12; -} + case INS_ldp: + case INS_ldpsw: + case INS_ldnp: + case INS_stp: + case INS_stnp: + emitIns_R_R_R_I(ins, attr, reg1, reg2, reg3, 0); + return; -/***************************************************************************** - * - * Returns an encoding for the condition code with the lowest bit inverted (marked by invert() in the - * architecture manual). - */ + case INS_stxr: + case INS_stxrb: + case INS_stxrh: + case INS_stlxr: + case INS_stlxrb: + case INS_stlxrh: + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrSP(reg3)); + fmt = IF_LS_3D; + break; -/*static*/ emitter::code_t emitter::insEncodeInvertedCond(insCond cond) -{ - emitter::code_t uimm = (emitter::code_t)cond; - uimm ^= 1; // invert the lowest bit - return uimm << 12; -} + case INS_casb: + case INS_casab: + case INS_casalb: + case INS_caslb: + case INS_cash: + case INS_casah: + case INS_casalh: + case INS_caslh: + case INS_cas: + case INS_casa: + case INS_casal: + case INS_casl: + case INS_ldaddb: + case INS_ldaddab: + case INS_ldaddalb: + case INS_ldaddlb: + case INS_ldaddh: + case INS_ldaddah: + case INS_ldaddalh: + case INS_ldaddlh: + case INS_ldadd: + case INS_ldadda: + case INS_ldaddal: + case INS_ldaddl: + case INS_ldclral: + case INS_ldsetal: + case INS_swpb: + case INS_swpab: + case INS_swpalb: + case INS_swplb: + case INS_swph: + case INS_swpah: + case INS_swpalh: + case INS_swplh: + case INS_swp: + case INS_swpa: + case INS_swpal: + case INS_swpl: + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrSP(reg3)); + fmt = IF_LS_3E; + break; -/***************************************************************************** - * - * Returns an encoding for the specified flags. - */ + case INS_sha256h: + case INS_sha256h2: + case INS_sha256su1: + case INS_sha1su0: + case INS_sha1c: + case INS_sha1p: + case INS_sha1m: + assert(isValidVectorDatasize(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (opt == INS_OPTS_NONE) + { + elemsize = EA_4BYTE; + opt = optMakeArrangement(size, elemsize); + } + assert(isValidArrangement(size, opt)); + fmt = IF_DV_3F; + break; -/*static*/ emitter::code_t emitter::insEncodeFlags(insCflags flags) -{ - emitter::code_t uimm = (emitter::code_t)flags; - return uimm; -} + case INS_ld2: + case INS_ld3: + case INS_ld4: + case INS_st2: + case INS_st3: + case INS_st4: + assert(opt != INS_OPTS_1D); // .1D format only permitted with LD1 & ST1 + FALLTHROUGH; -/***************************************************************************** - * - * Returns the encoding for the Shift Count bits to be used for Arm64 encodings - */ + case INS_ld1: + case INS_ld1_2regs: + case INS_ld1_3regs: + case INS_ld1_4regs: + case INS_st1: + case INS_st1_2regs: + case INS_st1_3regs: + case INS_st1_4regs: + case INS_ld1r: + case INS_ld2r: + case INS_ld3r: + case INS_ld4r: + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrSP(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidArrangement(size, opt)); -/*static*/ emitter::code_t emitter::insEncodeShiftCount(ssize_t imm, emitAttr size) -{ - assert((imm & 0x003F) == imm); - assert(((imm & 0x0020) == 0) || (size == EA_8BYTE)); + // Load/Store multiple structures post-indexed by a register + // Load single structure and replicate post-indexed by a register + reg2 = encodingSPtoZR(reg2); + fmt = IF_LS_3F; + break; - return (emitter::code_t)imm << 10; -} + case INS_addhn: + case INS_raddhn: + case INS_rsubhn: + case INS_subhn: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_8BYTE); + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_1D); // The encoding size = 11, Q = x is reserved. + fmt = IF_DV_3A; + break; -/***************************************************************************** - * - * Returns the encoding to select a 64-bit datasize for an Arm64 instruction - */ + case INS_addhn2: + case INS_raddhn2: + case INS_rsubhn2: + case INS_subhn2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_16BYTE); + assert(isValidArrangement(size, opt)); + assert(opt != INS_OPTS_2D); // The encoding size = 11, Q = x is reserved. + fmt = IF_DV_3A; + break; -/*static*/ emitter::code_t emitter::insEncodeDatasize(emitAttr size) -{ - if (size == EA_8BYTE) - { - return 0x80000000; // set the bit at location 31 - } - else - { - assert(size == EA_4BYTE); - return 0; - } -} + case INS_sabal: + case INS_sabdl: + case INS_saddl: + case INS_saddw: + case INS_smlal: + case INS_smlsl: + case INS_ssubl: + case INS_ssubw: + case INS_uabal: + case INS_uabdl: + case INS_uaddl: + case INS_uaddw: + case INS_umlal: + case INS_umlsl: + case INS_usubl: + case INS_usubw: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_8BYTE); + assert((opt == INS_OPTS_8B) || (opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); + fmt = IF_DV_3A; + break; -/***************************************************************************** - * - * Returns the encoding to select the datasize for the general load/store Arm64 instructions - * - */ + case INS_sabal2: + case INS_sabdl2: + case INS_saddl2: + case INS_saddw2: + case INS_smlal2: + case INS_smlsl2: + case INS_ssubl2: + case INS_ssubw2: + case INS_umlal2: + case INS_umlsl2: + case INS_smull2: + case INS_uabal2: + case INS_uabdl2: + case INS_uaddl2: + case INS_uaddw2: + case INS_usubl2: + case INS_umull2: + case INS_usubw2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_16BYTE); + assert((opt == INS_OPTS_16B) || (opt == INS_OPTS_8H) || (opt == INS_OPTS_4S)); + fmt = IF_DV_3A; + break; -/*static*/ emitter::code_t emitter::insEncodeDatasizeLS(emitter::code_t code, emitAttr size) -{ - bool exclusive = ((code & 0x35000000) == 0); - bool atomic = ((code & 0x31200C00) == 0x30200000); + case INS_sqdmlal: + case INS_sqdmlsl: + case INS_sqdmull: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(size == EA_8BYTE); + assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); + fmt = IF_DV_3A; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert((size == EA_2BYTE) || (size == EA_4BYTE)); + fmt = IF_DV_3E; + } + break; - if ((code & 0x00800000) && !exclusive && !atomic) // Is this a sign-extending opcode? (i.e. ldrsw, ldrsh, ldrsb) - { - if ((code & 0x80000000) == 0) // Is it a ldrsh or ldrsb and not ldrsw ? - { - if (EA_SIZE(size) != EA_8BYTE) // Do we need to encode the 32-bit Rt size bit? + case INS_sqdmulh: + case INS_sqrdmlah: + case INS_sqrdmlsh: + case INS_sqrdmulh: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (insOptsAnyArrangement(opt)) { - return 0x00400000; // set the bit at location 22 + // Vector operation + assert(isValidVectorDatasize(size)); + elemsize = optGetElemsize(opt); + assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)); + fmt = IF_DV_3A; } - } - } - else if (code & 0x80000000) // Is this a ldr/str/ldur/stur opcode? - { - if (EA_SIZE(size) == EA_8BYTE) // Do we need to encode the 64-bit size bit? - { - return 0x40000000; // set the bit at location 30 - } - } - return 0; -} + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert((size == EA_2BYTE) || (size == EA_4BYTE)); + fmt = IF_DV_3E; + } + break; -/***************************************************************************** - * - * Returns the encoding to select the datasize for the vector load/store Arm64 instructions - * - */ + case INS_sqdmlal2: + case INS_sqdmlsl2: + case INS_sqdmull2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_16BYTE); + assert((opt == INS_OPTS_8H) || (opt == INS_OPTS_4S)); + fmt = IF_DV_3A; + break; -/*static*/ emitter::code_t emitter::insEncodeDatasizeVLS(emitter::code_t code, emitAttr size) -{ - code_t result = 0; + case INS_pmul: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isValidArrangement(size, opt)); + assert((opt == INS_OPTS_8B) || (opt == INS_OPTS_16B)); + fmt = IF_DV_3A; + break; - // Check bit 29 - if ((code & 0x20000000) == 0) - { - // LDR literal + case INS_pmull: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_8BYTE); + assert((opt == INS_OPTS_8B) || (opt == INS_OPTS_1D)); + fmt = IF_DV_3A; + break; - if (size == EA_16BYTE) - { - // set the operation size in bit 31 - result = 0x80000000; - } - else if (size == EA_8BYTE) - { - // set the operation size in bit 30 - result = 0x40000000; - } - else - { - assert(size == EA_4BYTE); - // no bits are set - result = 0x00000000; - } - } - else - { - // LDR non-literal + case INS_pmull2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_16BYTE); + assert((opt == INS_OPTS_16B) || (opt == INS_OPTS_2D)); + fmt = IF_DV_3A; + break; - if (size == EA_16BYTE) - { - // The operation size in bits 31 and 30 are zero - // Bit 23 specifies a 128-bit Load/Store - result = 0x00800000; - } - else if (size == EA_8BYTE) - { - // set the operation size in bits 31 and 30 - result = 0xC0000000; - } - else if (size == EA_4BYTE) - { - // set the operation size in bit 31 - result = 0x80000000; - } - else if (size == EA_2BYTE) - { - // set the operation size in bit 30 - result = 0x40000000; - } - else - { - assert(size == EA_1BYTE); - // The operation size in bits 31 and 30 are zero - result = 0x00000000; - } - } + case INS_sdot: + case INS_udot: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(((size == EA_8BYTE) && (opt == INS_OPTS_2S)) || ((size == EA_16BYTE) && (opt == INS_OPTS_4S))); + fmt = IF_DV_3A; + break; - // Or in bit 26 to indicate a Vector register is used as 'target' - result |= 0x04000000; + default: + // fallback to emit SVE instructions. + return emitInsSve_R_R_R(ins, attr, reg1, reg2, reg3, opt, sopt); - return result; -} + } // end switch (ins) -/***************************************************************************** - * - * Returns the encoding to select the datasize for the vector load/store Arm64 instructions - * - */ + assert(fmt != IF_NONE); -/*static*/ emitter::code_t emitter::insEncodeDatasizeVPLS(emitter::code_t code, emitAttr size) -{ - code_t result = 0; + instrDesc* id = emitNewInstr(attr); - if (size == EA_16BYTE) - { - // The operation size in bits 31 and 30 are zero - // Bit 23 specifies a 128-bit Load/Store - result = 0x80000000; - } - else if (size == EA_8BYTE) - { - // set the operation size in bits 31 and 30 - result = 0x40000000; - } - else if (size == EA_4BYTE) - { - // set the operation size in bit 31 - result = 0x00000000; - } + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); - // Or in bit 26 to indicate a Vector register is used as 'target' - result |= 0x04000000; + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); - return result; + dispIns(id); + appendToCurIG(id); } -/***************************************************************************** - * - * Returns the encoding to set the size bit and the N bits for a 'bitfield' instruction - * - */ - -/*static*/ emitter::code_t emitter::insEncodeDatasizeBF(emitter::code_t code, emitAttr size) +//----------------------------------------------------------------------------------- +// emitIns_R_R_R_I_LdStPair: Add an instruction storing 2 registers into a memory +// (pointed by reg3) and the offset (immediate). +// +// Arguments: +// ins - The instruction code +// attr - The emit attribute for register 1 +// attr2 - The emit attribute for register 2 +// reg1 - Register 1 +// reg2 - Register 2 +// reg3 - Register 3 +// imm - Immediate offset, prior to scaling by operand size +// varx1 - LclVar number 1 +// varx2 - LclVar number 2 +// offs1 - Memory offset of lclvar number 1 +// offs2 - Memory offset of lclvar number 2 +// +void emitter::emitIns_R_R_R_I_LdStPair(instruction ins, + emitAttr attr, + emitAttr attr2, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + int varx1, + int varx2, + int offs1, + int offs2 DEBUG_ARG(unsigned var1RefsOffs) DEBUG_ARG(unsigned var2RefsOffs)) { - // is bit 30 equal to 0? - if ((code & 0x40000000) == 0) // is the opcode one of extr, sxtb, sxth or sxtw + assert((ins == INS_stp) || (ins == INS_ldp)); + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + unsigned scale = 0; + + // Is the target a vector register? + if (isVectorRegister(reg1)) { - if (size == EA_8BYTE) // Do we need to set the sf and N bits? - { - return 0x80400000; // set the sf-bit at location 31 and the N-bit at location 22 - } + assert(isValidVectorLSPDatasize(size)); + assert(isVectorRegister(reg2)); + + scale = NaturalScale_helper(size); + assert((scale >= 2) && (scale <= 4)); + } + else + { + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegisterOrZR(reg2)); + scale = (size == EA_8BYTE) ? 3 : 2; } - return 0; // don't set any bits -} -/***************************************************************************** - * - * Returns the encoding to select the 64/128-bit datasize for an Arm64 vector instruction - */ + reg3 = encodingSPtoZR(reg3); -/*static*/ emitter::code_t emitter::insEncodeVectorsize(emitAttr size) -{ - if (size == EA_16BYTE) + fmt = IF_LS_3C; + ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate + if (imm == 0) { - return 0x40000000; // set the bit at location 30 + fmt = IF_LS_3B; } else { - assert(size == EA_8BYTE); - return 0; + if ((imm & mask) == 0) + { + imm >>= scale; // The immediate is scaled by the size of the ld/st + } + else + { + // Unlike emitIns_S_S_R_R(), we would never come here when + // (imm & mask) != 0. + unreached(); + } } -} -/***************************************************************************** - * - * Returns the encoding to set the vector length specifier (vl) for an Arm64 SVE instruction - */ + bool validVar1 = varx1 != -1; + bool validVar2 = varx2 != -1; -/*static*/ emitter::code_t emitter::insEncodeVectorLengthSpecifier(instrDesc* id) -{ - assert(id != nullptr); - assert(insOptsScalableStandard(id->idInsOpt())); + instrDesc* id; + + if (validVar1 && validVar2) + { + id = emitNewInstrLclVarPair(attr, imm); + id->idAddr()->iiaLclVar.initLclVarAddr(varx1, offs1); + id->idSetIsLclVar(); - if (id->idVectorLength4x()) + emitGetLclVarPairLclVar2(id)->initLclVarAddr(varx2, offs2); + } + else { - switch (id->idInsFmt()) + id = emitNewInstrCns(attr, imm); + if (validVar1) { - case IF_SVE_DL_2A: - return 0x400; // set the bit at location 10 - case IF_SVE_DY_3A: - return 0x2000; // set the bit at location 13 - default: - assert(!"Unexpected format"); - break; + id->idAddr()->iiaLclVar.initLclVarAddr(varx1, offs1); + id->idSetIsLclVar(); + } + if (validVar2) + { + id->idAddr()->iiaLclVar.initLclVarAddr(varx2, offs2); + id->idSetIsLclVar(); } } - return 0; -} + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); -/***************************************************************************** - * - * Returns the encoding to select 'index' for an Arm64 vector elem instruction - */ -/*static*/ emitter::code_t emitter::insEncodeVectorIndex(emitAttr elemsize, ssize_t index) -{ - code_t bits = (code_t)index; - if (elemsize == EA_1BYTE) - { - bits <<= 1; - bits |= 1; - } - else if (elemsize == EA_2BYTE) + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + + // Record the attribute for the second register in the pair + if (EA_IS_GCREF(attr2)) { - bits <<= 2; - bits |= 2; + id->idGCrefReg2(GCT_GCREF); } - else if (elemsize == EA_4BYTE) + else if (EA_IS_BYREF(attr2)) { - bits <<= 3; - bits |= 4; + id->idGCrefReg2(GCT_BYREF); } else { - assert(elemsize == EA_8BYTE); - bits <<= 4; - bits |= 8; + id->idGCrefReg2(GCT_NONE); } - assert((bits >= 1) && (bits <= 0x1f)); - return (bits << 16); // bits at locations [20,19,18,17,16] +#ifdef DEBUG + id->idDebugOnlyInfo()->idVarRefOffs = var1RefsOffs; + id->idDebugOnlyInfo()->idVarRefOffs2 = var2RefsOffs; +#endif + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to select 'index2' for an Arm64 'ins' elem instruction + * Add an instruction referencing three registers and a constant. */ -/*static*/ emitter::code_t emitter::insEncodeVectorIndex2(emitAttr elemsize, ssize_t index2) + +void emitter::emitIns_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */, + emitAttr attrReg2 /* = EA_UNKNOWN */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) { - code_t bits = (code_t)index2; - if (elemsize == EA_1BYTE) - { - // bits are correct - } - else if (elemsize == EA_2BYTE) - { - bits <<= 1; - } - else if (elemsize == EA_4BYTE) - { - bits <<= 2; - } - else + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; + bool isLdSt = false; + bool isSIMD = false; + bool isAddSub = false; + bool setFlags = false; + unsigned scale = 0; + + /* Figure out the encoding format of the instruction */ + switch (ins) { - assert(elemsize == EA_8BYTE); - bits <<= 3; - } - assert((bits >= 0) && (bits <= 0xf)); + case INS_extr: + assert(insOptsNone(opt)); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidImmShift(imm, size)); + fmt = IF_DR_3E; + break; + + case INS_and: + case INS_ands: + case INS_eor: + case INS_orr: + case INS_bic: + case INS_bics: + case INS_eon: + case INS_orn: + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidImmShift(imm, size)); + if (imm == 0) + { + assert(insOptsNone(opt)); // a zero imm, means no shift kind + fmt = IF_DR_3A; + } + else + { + assert(insOptsAnyShift(opt)); // a non-zero imm, must select shift kind + fmt = IF_DR_3B; + } + break; + + case INS_fmul: // by element, imm[0..3] selects the element of reg3 + case INS_fmla: + case INS_fmls: + case INS_fmulx: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorElemsizeFloat(elemsize)); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + assert(opt != INS_OPTS_1D); // Reserved encoding + fmt = IF_DV_3BI; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert(isValidScalarDatasize(size)); + elemsize = size; + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + fmt = IF_DV_3DI; + } + break; + + case INS_mul: // by element, imm[0..7] selects the element of reg3 + case INS_mla: + case INS_mls: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + // Vector operation + assert(insOptsAnyArrangement(opt)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + elemsize = optGetElemsize(opt); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + // Only has encodings for H or S elemsize + assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)); + // Only has encodings for V0..V15 + if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) + { + noway_assert(!"Invalid reg3"); + } + fmt = IF_DV_3AI; + break; + + case INS_add: + case INS_sub: + setFlags = false; + isAddSub = true; + break; + + case INS_adds: + case INS_subs: + setFlags = true; + isAddSub = true; + break; + + case INS_ldpsw: + scale = 2; + isLdSt = true; + break; + + case INS_ldnp: + case INS_stnp: + assert(insOptsNone(opt)); // Can't use Pre/Post index on these two instructions + FALLTHROUGH; + + case INS_ldp: + case INS_stp: + // Is the target a vector register? + if (isVectorRegister(reg1)) + { + scale = NaturalScale_helper(size); + isSIMD = true; + } + else + { + scale = (size == EA_8BYTE) ? 3 : 2; + } + isLdSt = true; + fmt = IF_LS_3C; + break; - return (bits << 11); // bits at locations [14,13,12,11] -} + case INS_ld1: + case INS_ld2: + case INS_ld3: + case INS_ld4: + case INS_st1: + case INS_st2: + case INS_st3: + case INS_st4: + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrSP(reg2)); + assert(isGeneralRegister(reg3)); -/***************************************************************************** - * - * Returns the encoding for an immediate in the SVE variant of dup (indexed) - */ -/*static*/ emitter::code_t emitter::insEncodeSveBroadcastIndex(emitAttr elemsize, ssize_t index) -{ - unsigned lane_bytes = genLog2(elemsize) + 1; - code_t tsz = (1 << (lane_bytes - 1)); - code_t imm = (code_t)index << lane_bytes | tsz; - return insEncodeSplitUimm<23, 22, 20, 16>(imm); -} + assert(insOptsPostIndex(opt)); -/***************************************************************************** - * - * Returns the encoding to select the 'index' for an Arm64 'mul' by element instruction - */ -/*static*/ emitter::code_t emitter::insEncodeVectorIndexLMH(emitAttr elemsize, ssize_t index) -{ - code_t bits = 0; + elemsize = size; + assert(isValidVectorElemsize(elemsize)); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - if (elemsize == EA_2BYTE) - { - assert((index >= 0) && (index <= 7)); - if (index & 0x4) - { - bits |= (1 << 11); // set bit 11 'H' - } - if (index & 0x2) - { - bits |= (1 << 21); // set bit 21 'L' - } - if (index & 0x1) - { - bits |= (1 << 20); // set bit 20 'M' - } - } - else if (elemsize == EA_4BYTE) - { - assert((index >= 0) && (index <= 3)); - if (index & 0x2) - { - bits |= (1 << 11); // set bit 11 'H' - } - if (index & 0x1) - { - bits |= (1 << 21); // set bit 21 'L' - } - } - else - { - assert(!"Invalid 'elemsize' value"); - } + // Load/Store single structure post-indexed by a register + reg2 = encodingSPtoZR(reg2); + fmt = IF_LS_3G; + break; - return bits; -} + case INS_ext: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isValidVectorDatasize(size)); + assert(isValidArrangement(size, opt)); + assert((opt == INS_OPTS_8B) || (opt == INS_OPTS_16B)); + assert(isValidVectorIndex(size, EA_1BYTE, imm)); + fmt = IF_DV_3G; + break; -/***************************************************************************** - * - * Returns the encoding for a shift instruction, ready for insertion into an instruction. - */ -/*static*/ emitter::code_t emitter::insEncodeShiftImmediate(emitAttr size, bool isRightShift, ssize_t shiftAmount) -{ - if (isRightShift) - { - // The right shift amount must be in the range 1 to the destination element width in bits. - assert((shiftAmount > 0) && (shiftAmount <= getBitWidth(size))); - return (code_t)(2 * getBitWidth(size) - shiftAmount); - } - else - { - // The left shift amount must in the range 0 to the element width in bits minus 1. - assert(shiftAmount < getBitWidth(size)); - return (code_t)(getBitWidth(size) + shiftAmount); - } -} + case INS_smlal: + case INS_smlsl: + case INS_smull: + case INS_umlal: + case INS_umlsl: + case INS_umull: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_8BYTE); + assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); + elemsize = optGetElemsize(opt); + // Restricted to V0-V15 when element size is H. + if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) + { + assert(!"Invalid reg3"); + } + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + fmt = IF_DV_3AI; + break; -/***************************************************************************** - * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 vector instruction - */ + case INS_sqdmlal: + case INS_sqdmlsl: + case INS_sqdmull: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(size == EA_8BYTE); + assert((opt == INS_OPTS_4H) || (opt == INS_OPTS_2S)); + elemsize = optGetElemsize(opt); + fmt = IF_DV_3AI; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert((size == EA_2BYTE) || (size == EA_4BYTE)); + elemsize = size; + fmt = IF_DV_3EI; + } + // Restricted to V0-V15 when element size is H. + if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) + { + assert(!"Invalid reg3"); + } + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + break; -/*static*/ emitter::code_t emitter::insEncodeElemsize(emitAttr size) -{ - if (size == EA_8BYTE) - { - return 0x00C00000; // set the bit at location 23 and 22 - } - else if (size == EA_4BYTE) - { - return 0x00800000; // set the bit at location 23 - } - else if (size == EA_2BYTE) - { - return 0x00400000; // set the bit at location 22 - } - assert(size == EA_1BYTE); - return 0x00000000; -} + case INS_sqdmulh: + case INS_sqrdmlah: + case INS_sqrdmlsh: + case INS_sqrdmulh: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + if (insOptsAnyArrangement(opt)) + { + // Vector operation + assert(isValidVectorDatasize(size)); + elemsize = optGetElemsize(opt); + assert((elemsize == EA_2BYTE) || (elemsize == EA_4BYTE)); + fmt = IF_DV_3AI; + } + else + { + // Scalar operation + assert(insOptsNone(opt)); + assert((size == EA_2BYTE) || (size == EA_4BYTE)); + elemsize = size; + fmt = IF_DV_3EI; + } + // Restricted to V0-V15 when element size is H. + if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) + { + assert(!"Invalid reg3"); + } + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + break; -/***************************************************************************** - * - * Returns the encoding to select the 4/8 byte elemsize for an Arm64 float vector instruction - */ + case INS_smlal2: + case INS_smlsl2: + case INS_smull2: + case INS_sqdmlal2: + case INS_sqdmlsl2: + case INS_sqdmull2: + case INS_umlal2: + case INS_umlsl2: + case INS_umull2: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(size == EA_16BYTE); + assert((opt == INS_OPTS_8H) || (opt == INS_OPTS_4S)); + elemsize = optGetElemsize(opt); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + // Restricted to V0-V15 when element size is H + if ((elemsize == EA_2BYTE) && ((genRegMask(reg3) & RBM_ASIMD_INDEXED_H_ELEMENT_ALLOWED_REGS) == 0)) + { + assert(!"Invalid reg3"); + } + fmt = IF_DV_3AI; + break; -/*static*/ emitter::code_t emitter::insEncodeFloatElemsize(emitAttr size) -{ - if (size == EA_8BYTE) - { - return 0x00400000; // set the bit at location 22 - } - assert(size == EA_4BYTE); - return 0x00000000; -} + case INS_sdot: + case INS_udot: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(((size == EA_8BYTE) && (opt == INS_OPTS_2S)) || ((size == EA_16BYTE) && (opt == INS_OPTS_4S))); + assert(isValidVectorIndex(EA_16BYTE, EA_4BYTE, imm)); + fmt = IF_DV_3AI; + break; -// Returns the encoding to select the index for an Arm64 float vector by element instruction -/*static*/ emitter::code_t emitter::insEncodeFloatIndex(emitAttr elemsize, ssize_t index) -{ - code_t result = 0x00000000; - if (elemsize == EA_8BYTE) + default: + // fallback to emit SVE instructions. + return emitInsSve_R_R_R_I(ins, attr, reg1, reg2, reg3, imm, opt, sopt); + + } // end switch (ins) + + assert(insScalableOptsNone(sopt)); + + if (isLdSt) { - assert((index >= 0) && (index <= 1)); - if (index == 1) + assert(!isAddSub); + assert(isGeneralRegisterOrSP(reg3)); + assert(insOptsNone(opt) || insOptsIndexed(opt)); + + if (isSIMD) { - result |= 0x00000800; // 'H' - set the bit at location 11 + assert(isValidVectorLSPDatasize(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert((scale >= 2) && (scale <= 4)); } - } - else - { - assert(elemsize == EA_4BYTE); - assert((index >= 0) && (index <= 3)); - if (index & 2) + else { - result |= 0x00000800; // 'H' - set the bit at location 11 + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert((scale == 2) || (scale == 3)); } - if (index & 1) + + // Load/Store Pair reserved encodings: + if (emitInsIsLoad(ins)) { - result |= 0x00200000; // 'L' - set the bit at location 21 + assert(reg1 != reg2); + } + if (insOptsIndexed(opt)) + { + assert(reg1 != reg3); + assert(reg2 != reg3); } - } - return result; -} -/***************************************************************************** - * - * Returns the encoding to select the vector elemsize for an Arm64 ld/st# vector instruction - */ + reg3 = encodingSPtoZR(reg3); -/*static*/ emitter::code_t emitter::insEncodeVLSElemsize(emitAttr size) -{ - code_t result = 0x00000000; + ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate + if (imm == 0) + { + assert(insOptsNone(opt)); // PRE/POST Index doesn't make sense with an immediate of zero - switch (size) - { - case EA_1BYTE: + fmt = IF_LS_3B; + } + else { - result |= 0x0000; // clear bits 10 and 11 - break; + if ((imm & mask) == 0) + { + imm >>= scale; // The immediate is scaled by the size of the ld/st + + if ((imm >= -64) && (imm <= 63)) + { + fmt = IF_LS_3C; + } + } +#ifdef DEBUG + if (fmt != IF_LS_3C) + { + assert(!"Instruction cannot be encoded: IF_LS_3C"); + } +#endif } + } + else if (isAddSub) + { + bool reg2IsSP = (reg2 == REG_SP); + assert(!isLdSt); + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg3)); - case EA_2BYTE: + if (setFlags || insOptsAluShift(opt)) // Can't encode SP in reg1 with setFlags or AluShift option { - result |= 0x0400; // set bit at location 10, clear bit at location 11 - break; + assert(isGeneralRegisterOrZR(reg1)); } - - case EA_4BYTE: + else { - result |= 0x0800; // clear bit at location 10, set bit at location 11 - break; + assert(isGeneralRegisterOrSP(reg1)); + reg1 = encodingSPtoZR(reg1); } - case EA_8BYTE: + if (insOptsAluShift(opt)) // Can't encode SP in reg2 with AluShift option { - result |= 0x0C00; // set bits at location 10 and 11 - break; + assert(isGeneralRegister(reg2)); } - - default: + else { - assert(!"Invalid element size"); - break; + assert(isGeneralRegisterOrSP(reg2)); + reg2 = encodingSPtoZR(reg2); } - } - - return result; -} - -/***************************************************************************** - * - * Returns the encoding to select the index for an Arm64 ld/st# vector by element instruction - */ - -/*static*/ emitter::code_t emitter::insEncodeVLSIndex(emitAttr size, ssize_t index) -{ - code_t result = 0x00000000; - switch (size) - { - case EA_1BYTE: + if (insOptsAnyExtend(opt)) { - // Q = ? - bit location 30 - // xx = 00 - bit location 14 and 15 - // S = ? - bit location 12 - // ss = ?0 - bit location 10 and 11 + assert((imm >= 0) && (imm <= 4)); - result |= (index & 0x8) << 27; - result |= (index & 0x4) << 10; - result |= (index & 0x3) << 10; - break; + fmt = IF_DR_3C; } - - case EA_2BYTE: + else if (insOptsAluShift(opt)) { - // Q = ? - bit location 30 - // xx = 01 - bit location 14 and 15 - // S = ? - bit location 12 - // ss = ?? - bit location 10 and 11 - - result |= (index & 0x4) << 28; - result |= 0x4000; - result |= (index & 0x2) << 11; - result |= (index & 0x1) << 11; - break; + // imm should be non-zero and in [1..63] + assert(isValidImmShift(imm, size) && (imm != 0)); + fmt = IF_DR_3B; } - - case EA_4BYTE: + else if (imm == 0) { - // Q = ? - bit location 30 - // xx = 10 - bit location 14 and 15 - // S = ? - bit location 12 - // ss = 00 - bit location 10 and 11 + assert(insOptsNone(opt)); - result |= (index & 0x2) << 29; - result |= 0x8000; - result |= (index & 0x1) << 12; - break; + if (reg2IsSP) + { + // To encode the SP register as reg2 we must use the IF_DR_3C encoding + // and also specify a LSL of zero (imm == 0) + opt = INS_OPTS_LSL; + fmt = IF_DR_3C; + } + else + { + fmt = IF_DR_3A; + } } - - case EA_8BYTE: + else { - // Q = ? - bit location 30 - // xx = 10 - bit location 14 and 15 - // S = 0 - bit location 12 - // ss = 01 - bit location 10 and 11 - - result |= (index & 0x1) << 30; - result |= 0x8400; - break; + assert(!"Instruction cannot be encoded: Add/Sub IF_DR_3A"); } + } - default: + assert(fmt != IF_NONE); + + instrDesc* id = emitNewInstrCns(attr, imm); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + + // Record the attribute for the second register in the pair + id->idGCrefReg2(GCT_NONE); + if (attrReg2 != EA_UNKNOWN) + { + // Record the attribute for the second register in the pair + assert((fmt == IF_LS_3B) || (fmt == IF_LS_3C)); + if (EA_IS_GCREF(attrReg2)) { - assert(!"Invalid element size"); - break; + id->idGCrefReg2(GCT_GCREF); + } + else if (EA_IS_BYREF(attrReg2)) + { + id->idGCrefReg2(GCT_BYREF); } } - return result; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to select the fcvt operation for Arm64 instructions + * Add an instruction referencing three registers and two constants. */ -/*static*/ emitter::code_t emitter::insEncodeConvertOpt(insFormat fmt, insOpts conversion) -{ - code_t result = 0; - switch (conversion) - { - case INS_OPTS_S_TO_D: // Single to Double - assert(fmt == IF_DV_2J); - result = 0x00008000; // type=00, opc=01 - break; - - case INS_OPTS_D_TO_S: // Double to Single - assert(fmt == IF_DV_2J); - result = 0x00400000; // type=01, opc=00 - break; - - case INS_OPTS_H_TO_S: // Half to Single - assert(fmt == IF_DV_2J); - result = 0x00C00000; // type=11, opc=00 - break; - - case INS_OPTS_H_TO_D: // Half to Double - assert(fmt == IF_DV_2J); - result = 0x00C08000; // type=11, opc=01 - break; - case INS_OPTS_S_TO_H: // Single to Half - assert(fmt == IF_DV_2J); - result = 0x00018000; // type=00, opc=11 - break; - - case INS_OPTS_D_TO_H: // Double to Half - assert(fmt == IF_DV_2J); - result = 0x00418000; // type=01, opc=11 - break; - - case INS_OPTS_S_TO_4BYTE: // Single to INT32 - assert(fmt == IF_DV_2H); - result = 0x00000000; // sf=0, type=00 - break; +void emitter::emitIns_R_R_R_I_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm1, + ssize_t imm2, + insOpts opt) +{ + // Currently, only SVE instructions use this format. + emitInsSve_R_R_R_I_I(ins, attr, reg1, reg2, reg3, imm1, imm2, opt); +} - case INS_OPTS_D_TO_4BYTE: // Double to INT32 - assert(fmt == IF_DV_2H); - result = 0x00400000; // sf=0, type=01 - break; +/***************************************************************************** + * + * Add an instruction referencing three registers, with an extend option + */ - case INS_OPTS_S_TO_8BYTE: // Single to INT64 - assert(fmt == IF_DV_2H); - result = 0x80000000; // sf=1, type=00 - break; +void emitter::emitIns_R_R_R_Ext(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + insOpts opt, /* = INS_OPTS_NONE */ + int shiftAmount) /* = -1 -- unset */ +{ + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + bool isSIMD = false; + int scale = -1; - case INS_OPTS_D_TO_8BYTE: // Double to INT64 - assert(fmt == IF_DV_2H); - result = 0x80400000; // sf=1, type=01 + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_ldrb: + case INS_ldrsb: + case INS_strb: + scale = 0; break; - case INS_OPTS_4BYTE_TO_S: // INT32 to Single - assert(fmt == IF_DV_2I); - result = 0x00000000; // sf=0, type=00 + case INS_ldrh: + case INS_ldrsh: + case INS_strh: + scale = 1; break; - case INS_OPTS_4BYTE_TO_D: // INT32 to Double - assert(fmt == IF_DV_2I); - result = 0x00400000; // sf=0, type=01 + case INS_ldrsw: + scale = 2; break; - case INS_OPTS_8BYTE_TO_S: // INT64 to Single - assert(fmt == IF_DV_2I); - result = 0x80000000; // sf=1, type=00 - break; + case INS_ldr: + case INS_str: + // Is the target a vector register? + if (isVectorRegister(reg1)) + { + assert(isValidVectorLSDatasize(size)); + scale = NaturalScale_helper(size); + isSIMD = true; + } + else + { + assert(isValidGeneralDatasize(size)); + scale = (size == EA_8BYTE) ? 3 : 2; + } - case INS_OPTS_8BYTE_TO_D: // INT64 to Double - assert(fmt == IF_DV_2I); - result = 0x80400000; // sf=1, type=01 break; default: - assert(!"Invalid 'conversion' value"); + unreached(); break; - } - return result; -} -/***************************************************************************** - * - * Returns the encoding to have the Rn register be updated Pre/Post indexed - * or not updated - */ + } // end switch (ins) -/*static*/ emitter::code_t emitter::insEncodeIndexedOpt(insOpts opt) -{ - assert(emitter::insOptsNone(opt) || emitter::insOptsIndexed(opt)); + assert(scale != -1); + assert(insOptsLSExtend(opt)); - if (emitter::insOptsIndexed(opt)) + if (isSIMD) { - if (emitter::insOptsPostIndex(opt)) - { - return 0x00000400; // set the bit at location 10 - } - else - { - assert(emitter::insOptsPreIndex(opt)); - return 0x00000C00; // set the bit at location 10 and 11 - } + assert(isValidVectorLSDatasize(size)); + assert(isVectorRegister(reg1)); } else { - assert(emitter::insOptsNone(opt)); - return 0; // bits 10 and 11 are zero + assert(isValidGeneralLSDatasize(size)); + assert(isGeneralRegisterOrZR(reg1)); } -} - -/***************************************************************************** - * - * Returns the encoding for a ldp/stp instruction to have the Rn register - * be updated Pre/Post indexed or not updated - */ -/*static*/ emitter::code_t emitter::insEncodePairIndexedOpt(instruction ins, insOpts opt) -{ - assert(emitter::insOptsNone(opt) || emitter::insOptsIndexed(opt)); + assert(isGeneralRegisterOrSP(reg2)); + assert(isGeneralRegister(reg3)); - if ((ins == INS_ldnp) || (ins == INS_stnp)) + // Load/Store reserved encodings: + if (insOptsIndexed(opt)) { - assert(emitter::insOptsNone(opt)); - return 0; // bits 23 and 24 are zero + assert(reg1 != reg2); } - else + + if (shiftAmount == -1) { - if (emitter::insOptsIndexed(opt)) - { - if (emitter::insOptsPostIndex(opt)) - { - return 0x00800000; // set the bit at location 23 - } - else - { - assert(emitter::insOptsPreIndex(opt)); - return 0x01800000; // set the bit at location 24 and 23 - } - } - else - { - assert(emitter::insOptsNone(opt)); - return 0x01000000; // set the bit at location 24 - } + shiftAmount = insOptsLSL(opt) ? scale : 0; } -} -/***************************************************************************** - * - * Returns the encoding to apply a Shift Type on the Rm register - */ + assert((shiftAmount == scale) || (shiftAmount == 0)); -/*static*/ emitter::code_t emitter::insEncodeShiftType(insOpts opt) -{ - if (emitter::insOptsNone(opt)) - { - // None implies the we encode LSL (with a zero immediate) - opt = INS_OPTS_LSL; - } - assert(emitter::insOptsAnyShift(opt)); + reg2 = encodingSPtoZR(reg2); + fmt = IF_LS_3A; - emitter::code_t option = (emitter::code_t)opt - (emitter::code_t)INS_OPTS_LSL; - assert(option <= 3); + instrDesc* id = emitNewInstr(attr); - return option << 22; // bits 23, 22 -} + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); -/***************************************************************************** - * - * Returns the encoding to apply a 12 bit left shift to the immediate - */ + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + id->idReg3Scaled(shiftAmount == scale); -/*static*/ emitter::code_t emitter::insEncodeShiftImm12(insOpts opt) -{ - if (emitter::insOptsLSL12(opt)) - { - return 0x00400000; // set the bit at location 22 - } - return 0; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to have the Rm register use an extend operation + * Add an instruction referencing two registers and two constants. */ -/*static*/ emitter::code_t emitter::insEncodeExtend(insOpts opt) +void emitter::emitIns_R_R_I_I( + instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, int imm1, int imm2, insOpts opt) { - if (emitter::insOptsNone(opt) || (opt == INS_OPTS_LSL)) + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; + size_t immOut = 0; // composed from imm1 and imm2 and stored in the instrDesc + + /* Figure out the encoding format of the instruction */ + switch (ins) { - // None or LSL implies the we encode UXTX - opt = INS_OPTS_UXTX; - } - assert(emitter::insOptsAnyExtend(opt)); + int lsb; + int width; + bitMaskImm bmi; + unsigned registerListSize; - emitter::code_t option = (emitter::code_t)opt - (emitter::code_t)INS_OPTS_UXTB; - assert(option <= 7); + case INS_bfm: + case INS_sbfm: + case INS_ubfm: + assert(isGeneralRegister(reg1)); + assert((ins == INS_bfm) ? isGeneralRegisterOrZR(reg2) : isGeneralRegister(reg2)); + assert(isValidImmShift(imm1, size)); + assert(isValidImmShift(imm2, size)); + assert(insOptsNone(opt)); + bmi.immNRS = 0; + bmi.immN = (size == EA_8BYTE); + bmi.immR = imm1; + bmi.immS = imm2; + immOut = bmi.immNRS; + fmt = IF_DI_2D; + break; + + case INS_bfi: + case INS_sbfiz: + case INS_ubfiz: + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + lsb = getBitWidth(size) - imm1; + width = imm2 - 1; + assert(isValidImmShift(lsb, size)); + assert(isValidImmShift(width, size)); + assert(insOptsNone(opt)); + bmi.immNRS = 0; + bmi.immN = (size == EA_8BYTE); + bmi.immR = lsb; + bmi.immS = width; + immOut = bmi.immNRS; + fmt = IF_DI_2D; + break; + + case INS_bfxil: + case INS_sbfx: + case INS_ubfx: + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + lsb = imm1; + width = imm2 + imm1 - 1; + assert(isValidImmShift(lsb, size)); + assert(isValidImmShift(width, size)); + assert(insOptsNone(opt)); + bmi.immNRS = 0; + bmi.immN = (size == EA_8BYTE); + bmi.immR = imm1; + bmi.immS = imm2 + imm1 - 1; + immOut = bmi.immNRS; + fmt = IF_DI_2D; + break; - return option << 13; // bits 15,14,13 -} + case INS_mov: + case INS_ins: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + elemsize = size; + assert(isValidVectorElemsize(elemsize)); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm1)); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm2)); + assert(insOptsNone(opt)); + immOut = (imm1 << 4) + imm2; + fmt = IF_DV_2F; + break; -/***************************************************************************** - * - * Returns the encoding to scale the Rm register by {0,1,2,3,4} - * when using an extend operation - */ + case INS_ld1: + case INS_ld2: + case INS_ld3: + case INS_ld4: + case INS_st1: + case INS_st2: + case INS_st3: + case INS_st4: + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrSP(reg2)); -/*static*/ emitter::code_t emitter::insEncodeExtendScale(ssize_t imm) -{ - assert((imm >= 0) && (imm <= 4)); + elemsize = size; + assert(isValidVectorElemsize(elemsize)); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm1)); - return (emitter::code_t)imm << 10; // bits 12,11,10 -} + registerListSize = insGetRegisterListSize(ins); + assert((elemsize * registerListSize) == (unsigned)imm2); + assert(insOptsPostIndex(opt)); -/***************************************************************************** - * - * Returns the encoding to have the Rm register be auto scaled by the ld/st size - */ + // Load/Store single structure post-indexed by an immediate + reg2 = encodingSPtoZR(reg2); + immOut = imm1; + fmt = IF_LS_2G; + break; -/*static*/ emitter::code_t emitter::insEncodeReg3Scale(bool isScaled) -{ - if (isScaled) - { - return 0x00001000; // set the bit at location 12 - } - else - { - return 0; + default: + unreached(); + break; } -} - -/***************************************************************************** - * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - */ - -/*static*/ emitter::code_t emitter::insEncodeSveElemsize(emitAttr size) -{ - switch (size) - { - case EA_1BYTE: - return 0x00000000; + assert(fmt != IF_NONE); - case EA_2BYTE: - return 0x00400000; // set the bit at location 22 + instrDesc* id = emitNewInstrSC(attr, immOut); - case EA_4BYTE: - return 0x00800000; // set the bit at location 23 + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); - case EA_8BYTE: - return 0x00C00000; // set the bit at location 23 and 22 + id->idReg1(reg1); + id->idReg2(reg2); - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the size at bit locations '22-21'. + * Add an instruction referencing four registers. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_22_to_21(emitAttr size) +void emitter::emitIns_R_R_R_R(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + regNumber reg4, + insOpts opt /* = INS_OPTS_NONE*/, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) { - switch (size) - { - case EA_1BYTE: - return 0; + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; - case EA_2BYTE: - return (1 << 21); // set the bit at location 21 + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_madd: + case INS_msub: + case INS_smaddl: + case INS_smsubl: + case INS_umaddl: + case INS_umsubl: + assert(isValidGeneralDatasize(size)); + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isGeneralRegister(reg4)); + assert(insScalableOptsNone(sopt)); + fmt = IF_DR_4A; + break; - case EA_4BYTE: - return (1 << 22); // set the bit at location 22 + case INS_fmadd: + case INS_fmsub: + case INS_fnmadd: + case INS_fnmsub: + // Scalar operation + assert(isValidScalarDatasize(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isVectorRegister(reg4)); + assert(insScalableOptsNone(sopt)); + fmt = IF_DV_4A; + break; - case EA_8BYTE: - return (1 << 22) | (1 << 21); // set the bit at location 22 and 21 + case INS_invalid: + fmt = IF_NONE; + break; + // Fallback handles emitting the SVE instructions. default: - assert(!"Invalid insOpt for vector register"); + return emitInsSve_R_R_R_R(ins, attr, reg1, reg2, reg3, reg4, opt, sopt); } - return 0; -} - -/***************************************************************************** - * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the size at bit locations '18-17'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_18_to_17(emitAttr size) -{ - switch (size) - { - case EA_1BYTE: - return 0; + assert(fmt != IF_NONE); - case EA_2BYTE: - return (1 << 17); // set the bit at location 17 + instrDesc* id = emitNewInstr(attr); - case EA_4BYTE: - return (1 << 18); // set the bit at location 18 + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); - case EA_8BYTE: - return (1 << 18) | (1 << 17); // set the bit at location 18 and 17 + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + id->idReg4(reg4); - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'sz' at bit location '20'. + * Add an instruction referencing four registers and a constant. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_20(emitAttr size) +void emitter::emitIns_R_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + regNumber reg4, + ssize_t imm, + insOpts opt /* = INS_OPT_NONE*/) { - switch (size) - { - case EA_4BYTE: - return 0; - - case EA_8BYTE: - return (1 << 20); - - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; + // Currently, only SVE instructions use this format. + emitInsSve_R_R_R_R_I(ins, attr, reg1, reg2, reg3, reg4, imm, opt); } /***************************************************************************** * - * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'sz' at bit location '21'. + * Add an instruction referencing a register and a condition code */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_21(emitAttr size) +void emitter::emitIns_R_COND(instruction ins, emitAttr attr, regNumber reg, insCond cond) { - switch (size) - { - case EA_4BYTE: - return 0; + insFormat fmt = IF_NONE; + condFlagsImm cfi; + cfi.immCFVal = 0; - case EA_8BYTE: - return (1 << 21); + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_cset: + case INS_csetm: + assert(isGeneralRegister(reg)); + cfi.cond = cond; + fmt = IF_DR_1D; + break; default: - assert(!"Invalid insOpt for vector register"); - } - return 0; -} + unreached(); + break; -/***************************************************************************** - * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:20-19'. - */ + } // end switch (ins) -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_23_tszl_20_to_19(emitAttr size) -{ - switch (size) - { - case EA_1BYTE: - return 0x080000; // set the bit at location 19 + assert(fmt != IF_NONE); + assert(isValidImmCond(cfi.immCFVal)); - case EA_2BYTE: - return 0x100000; // set the bit at location 20 + instrDesc* id = emitNewInstrSC(attr, cfi.immCFVal); - case EA_4BYTE: - return 0x400000; // set the bit at location 22 + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); - case EA_8BYTE: - return 0x800000; // set the bit at location 23 + id->idReg1(reg); - default: - assert(!"Invalid size for vector register"); - } - return 0; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction at bit location '30'. - * This only works on select formats. + * Add an instruction referencing two registers and a condition code */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_30_or_21(insFormat fmt, emitAttr size) +void emitter::emitIns_R_R_COND(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, insCond cond) { - switch (fmt) - { - case IF_SVE_HX_3A_B: - case IF_SVE_HX_3A_E: - switch (size) - { - case EA_4BYTE: - return 0; - - case EA_8BYTE: - return (1 << 30); - - default: - break; - } - - assert(!"Invalid size for vector register"); - return 0; + insFormat fmt = IF_NONE; + condFlagsImm cfi; + cfi.immCFVal = 0; - case IF_SVE_IV_3A: - assert(size == EA_8BYTE); - return 0; + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_cinc: + case INS_cinv: + case INS_cneg: + assert(isGeneralRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + cfi.cond = cond; + fmt = IF_DR_2D; + break; + default: + unreached(); + break; - case IF_SVE_JI_3A_A: - switch (size) - { - case EA_4BYTE: - return (1 << 21); + } // end switch (ins) - case EA_8BYTE: - return 0; + assert(fmt != IF_NONE); + assert(isValidImmCond(cfi.immCFVal)); - default: - break; - } + instrDesc* id = emitNewInstrSC(attr, cfi.immCFVal); - assert(!"Invalid size for vector register"); - return 0; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); - default: - break; - } + id->idReg1(reg1); + id->idReg2(reg2); - assert(!"Unexpected instruction format"); - return 0; + dispIns(id); + appendToCurIG(id); } + /***************************************************************************** * - * Returns the encoding for the field 'i1:tszh:tszl' at bit locations '23-22:20-18'. + * Add an instruction referencing two registers and a condition code */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_tszl_and_imm(const insOpts opt, const ssize_t imm) +void emitter::emitIns_R_R_R_COND( + instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, regNumber reg3, insCond cond) { - code_t encoding = 0; + insFormat fmt = IF_NONE; + condFlagsImm cfi; + cfi.immCFVal = 0; - switch (opt) + /* Figure out the encoding format of the instruction */ + switch (ins) { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); - encoding = 0x040000; // set the bit at location 18 - // encode immediate at location 23-22:20-19 - encoding |= ((imm & 0b1100) << 22); - encoding |= ((imm & 0b11) << 19); + case INS_csel: + case INS_csinc: + case INS_csinv: + case INS_csneg: + assert(isGeneralRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrZR(reg3)); + cfi.cond = cond; + fmt = IF_DR_3D; break; - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); - encoding = 0x080000; // set the bit at location 19 - // encode immediate at location 23-22:20 - encoding |= ((imm & 0b110) << 22); - encoding |= ((imm & 1) << 20); + default: + unreached(); break; - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - encoding = 0x100000; // set the bit at location 20 - encoding |= (imm << 22); // encode immediate at location 23:22 - break; + } // end switch (ins) - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); - encoding = 0x400000; // set the bit at location 22 - encoding |= (imm << 23); // encode immediate at location 23 - break; + assert(fmt != IF_NONE); + assert(isValidImmCond(cfi.immCFVal)); - default: - assert(!"Invalid size for vector register"); - break; - } + instrDesc* id = emitNewInstr(attr); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + id->idSmallCns(cfi.immCFVal); - return encoding; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding for the field 'tszh:tszl:imm3' at bit locations '23-22:20-19:18-16'. + * Add an instruction referencing two registers the flags and a condition code */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithShift_tszh_tszl_imm3(const insOpts opt, - ssize_t imm, - bool isRightShift) +void emitter::emitIns_R_R_FLAGS_COND( + instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, insCflags flags, insCond cond) { - code_t encoding = 0; - - imm = insEncodeShiftImmediate(optGetSveElemsize(opt), isRightShift, imm); + insFormat fmt = IF_NONE; + condFlagsImm cfi; + cfi.immCFVal = 0; - switch (opt) + /* Figure out the encoding format of the instruction */ + switch (ins) { - case INS_OPTS_SCALABLE_B: - imm = imm & 0b111; // bits 18-16 - encoding |= (1 << 19); // bit 19 + case INS_ccmp: + case INS_ccmn: + assert(isGeneralRegister(reg1)); + assert(isGeneralRegister(reg2)); + cfi.flags = flags; + cfi.cond = cond; + fmt = IF_DR_2I; break; - - case INS_OPTS_SCALABLE_H: - imm = imm & 0b1111; // bits 19-16 - encoding |= (1 << 20); // bit 20 + default: + unreached(); break; + } // end switch (ins) - case INS_OPTS_SCALABLE_S: - imm = imm & 0b11111; // bits 20-16 - encoding |= (1 << 22); // bit 22 - break; + assert(fmt != IF_NONE); + assert(isValidImmCondFlags(cfi.immCFVal)); - case INS_OPTS_SCALABLE_D: - // this gets the last bit of 'imm' and tries to set bit 22 - encoding |= ((imm >> 5) << 22); - imm = imm & 0b11111; // bits 20-16 - encoding |= (1 << 23); // bit 23 - break; + instrDesc* id = emitNewInstrSC(attr, cfi.immCFVal); - default: - assert(!"Invalid size for vector register"); - break; - } + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); + + id->idReg1(reg1); + id->idReg2(reg2); - return (encoding | (code_t)(imm << 16)); + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding for the field 'i1:tsz' at bit locations '20:19-16'. + * Add an instruction referencing a register, an immediate, the flags and a condition code */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithImmediate_i1_tsz(const insOpts opt, ssize_t imm) +void emitter::emitIns_R_I_FLAGS_COND( + instruction ins, emitAttr attr, regNumber reg, ssize_t imm, insCflags flags, insCond cond) { - code_t encoding = 0; + insFormat fmt = IF_NONE; + condFlagsImm cfi; + cfi.immCFVal = 0; - switch (opt) + /* Figure out the encoding format of the instruction */ + switch (ins) { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); - encoding |= (1 << 16); // bit 16 - encoding |= (imm << 17); // bits 20-17 + case INS_ccmp: + case INS_ccmn: + assert(isGeneralRegister(reg)); + if (imm < 0) + { + ins = insReverse(ins); + imm = -imm; + } + if (isValidUimm<5>(imm)) + { + cfi.imm5 = imm; + cfi.flags = flags; + cfi.cond = cond; + fmt = IF_DI_1F; + } + else + { + assert(!"Instruction cannot be encoded: ccmp/ccmn imm5"); + } break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); - encoding |= (1 << 17); // bit 17 - encoding |= (imm << 18); // bits 20-18 + default: + unreached(); break; + } // end switch (ins) - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - encoding |= (1 << 18); // bit 18 - encoding |= (imm << 19); // bits 20-19 - break; + assert(fmt != IF_NONE); + assert(isValidImmCondFlagsImm5(cfi.immCFVal)); - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); - encoding |= (1 << 19); // bit 19 - encoding |= (imm << 20); // bit 20 - break; + instrDesc* id = emitNewInstrSC(attr, cfi.immCFVal); - default: - assert(!"Invalid size for vector register"); - break; - } + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); + + id->idReg1(reg); - return encoding; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to select the elemsize for an Arm64 SVE vector instruction plus an immediate. - * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:9-8'. + * Add a memory barrier instruction with a 'barrier' immediate */ -/*static*/ emitter::code_t emitter::insEncodeSveShift_23_to_22_9_to_0(emitAttr size, bool isRightShift, size_t imm) +void emitter::emitIns_BARR(instruction ins, insBarrier barrier) { - code_t encodedSize = 0; + insFormat fmt = IF_NONE; + ssize_t imm = 0; - switch (size) + /* Figure out the encoding format of the instruction */ + switch (ins) { - case EA_1BYTE: - encodedSize = 0x100; // set the bit at location 8 - break; + case INS_dsb: + case INS_dmb: + case INS_isb: - case EA_2BYTE: - encodedSize = 0x200; // set the bit at location 9 + fmt = IF_SI_0B; + imm = (ssize_t)barrier; break; - - case EA_4BYTE: - encodedSize = 0x400000; // set the bit at location 22 + default: + unreached(); break; + } // end switch (ins) - case EA_8BYTE: - encodedSize = 0x800000; // set the bit at location 23 - break; + assert(fmt != IF_NONE); - default: - assert(!"Invalid esize for vector register"); - } + instrDesc* id = emitNewInstrSC(EA_8BYTE, imm); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); - code_t encodedImm = insEncodeShiftImmediate(size, isRightShift, imm); - code_t imm3High = (encodedImm & 0x60) << 17; - code_t imm3Low = (encodedImm & 0x1f) << 5; - return encodedSize | imm3High | imm3Low; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to select the constant values 90 or 270 for an Arm64 SVE vector instruction - * This specifically encode the field 'rot' at bit location '16'. + * Add an instruction with a static data member operand. If 'size' is 0, the + * instruction operates on the address of the static member instead of its + * value (e.g. "push offset clsvar", rather than "push dword ptr [clsvar]"). */ -/*static*/ emitter::code_t emitter::insEncodeSveImm90_or_270_rot(ssize_t imm) +void emitter::emitIns_C(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE fldHnd, int offs) { - assert(emitIsValidEncodedRotationImm90_or_270(imm)); - return (code_t)(imm << 16); + NYI("emitIns_C"); } /***************************************************************************** * - * Returns the encoding to select the constant values 0, 90, 180 or 270 for an Arm64 SVE vector instruction - * This specifically encode the field 'rot' at bit locations '14-13'. + * Add an instruction referencing stack-based local variable. */ -/*static*/ emitter::code_t emitter::insEncodeSveImm0_to_270_rot(ssize_t imm) +void emitter::emitIns_S(instruction ins, emitAttr attr, int varx, int offs) { - assert(emitIsValidEncodedRotationImm0_to_270(imm)); - return (code_t)(imm << 13); + NYI("emitIns_S"); } /***************************************************************************** * - * Returns the encoding to select the constant float values 0, 0.5, 1.0 or 2.0 for an Arm64 SVE vector instruction - * This specifically encode the field 'i1' at bit location '5'. + * Add an instruction referencing a register and a stack-based local variable. */ +void emitter::emitIns_R_S(instruction ins, emitAttr attr, regNumber reg1, int varx, int offs) +{ + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + int disp = 0; + unsigned scale = 0; + bool isLdrStr = false; + + assert(offs >= 0); + + // TODO-ARM64-CQ: use unscaled loads? + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_strb: + case INS_ldrb: + case INS_ldrsb: + scale = 0; + break; + + case INS_strh: + case INS_ldrh: + case INS_ldrsh: + scale = 1; + break; + + case INS_ldrsw: + scale = 2; + break; + + case INS_str: + case INS_ldr: + assert(isValidGeneralDatasize(size) || isValidVectorDatasize(size)); + scale = genLog2(EA_SIZE_IN_BYTES(size)); + isLdrStr = true; + break; + + case INS_lea: + assert(size == EA_8BYTE); + scale = 0; + break; -/*static*/ emitter::code_t emitter::insEncodeSveSmallFloatImm(ssize_t imm) -{ - assert(emitIsValidEncodedSmallFloatImm(imm)); - return (code_t)(imm << 5); -} + default: + NYI("emitIns_R_S"); // FP locals? + return; -/***************************************************************************** - * - * Returns the register list size for the given SVE instruction. - */ + } // end switch (ins) -/*static*/ int emitter::insGetSveReg1ListSize(instruction ins) -{ - switch (ins) - { - case INS_sve_ld1d: - case INS_sve_ld1w: - case INS_sve_ld1sw: - case INS_sve_ld1sb: - case INS_sve_ld1b: - case INS_sve_ld1sh: - case INS_sve_ld1h: - case INS_sve_ldnf1d: - case INS_sve_ldnf1sw: - case INS_sve_ldnf1sh: - case INS_sve_ldnf1w: - case INS_sve_ldnf1h: - case INS_sve_ldnf1sb: - case INS_sve_ldnf1b: - case INS_sve_ldnt1b: - case INS_sve_ldnt1d: - case INS_sve_ldnt1h: - case INS_sve_ldnt1w: - case INS_sve_ld1rob: - case INS_sve_ld1rod: - case INS_sve_ld1roh: - case INS_sve_ld1row: - case INS_sve_ld1rqb: - case INS_sve_ld1rqd: - case INS_sve_ld1rqh: - case INS_sve_ld1rqw: - case INS_sve_stnt1b: - case INS_sve_stnt1d: - case INS_sve_stnt1h: - case INS_sve_stnt1w: - case INS_sve_st1d: - case INS_sve_st1w: - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - case INS_sve_ldff1h: - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - case INS_sve_st1b: - case INS_sve_st1h: - case INS_sve_ldff1sb: - case INS_sve_ldff1b: - case INS_sve_ldnt1sb: - case INS_sve_ldnt1sh: - case INS_sve_ld1rd: - case INS_sve_ld1rsw: - case INS_sve_ld1rh: - case INS_sve_ld1rsb: - case INS_sve_ld1rsh: - case INS_sve_ld1rw: - case INS_sve_ld1q: - case INS_sve_ldnt1sw: - case INS_sve_st1q: - case INS_sve_ld1rb: - return 1; - - case INS_sve_ld2b: - case INS_sve_ld2h: - case INS_sve_ld2w: - case INS_sve_ld2d: - case INS_sve_ld2q: - case INS_sve_splice: // SVE_CV_3A - case INS_sve_st2b: - case INS_sve_st2h: - case INS_sve_st2w: - case INS_sve_st2d: - case INS_sve_st2q: - case INS_sve_whilege: // SVE_DX_3A - case INS_sve_whilegt: // SVE_DX_3A - case INS_sve_whilehi: // SVE_DX_3A - case INS_sve_whilehs: // SVE_DX_3A - case INS_sve_whilele: // SVE_DX_3A - case INS_sve_whilels: // SVE_DX_3A - case INS_sve_whilelt: // SVE_DX_3A - case INS_sve_pext: // SVE_DW_2B - return 2; - - case INS_sve_ld3b: - case INS_sve_ld3h: - case INS_sve_ld3w: - case INS_sve_ld3d: - case INS_sve_ld3q: - case INS_sve_st3b: - case INS_sve_st3h: - case INS_sve_st3w: - case INS_sve_st3d: - case INS_sve_st3q: - return 3; - - case INS_sve_ld4b: - case INS_sve_ld4h: - case INS_sve_ld4w: - case INS_sve_ld4d: - case INS_sve_ld4q: - case INS_sve_st4b: - case INS_sve_st4h: - case INS_sve_st4w: - case INS_sve_st4d: - case INS_sve_st4q: - return 4; + /* Figure out the variable's frame position */ + ssize_t imm; + int base; + bool FPbased; - default: - assert(!"Unexpected instruction"); - return 1; - } -} + base = emitComp->lvaFrameAddress(varx, &FPbased); + disp = base + offs; + assert((scale >= 0) && (scale <= 4)); -/***************************************************************************** - * - * Returns the predicate type for the given SVE format. - */ + bool useRegForImm = false; + regNumber reg2 = FPbased ? REG_FPBASE : REG_SPBASE; + reg2 = encodingSPtoZR(reg2); -/*static*/ emitter::PredicateType emitter::insGetPredicateType(insFormat fmt, int regpos /* =0 */) -{ - switch (fmt) + if (ins == INS_lea) { - case IF_SVE_BV_2A: - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - case IF_SVE_HW_4A_B: - case IF_SVE_HW_4A_C: - case IF_SVE_HW_4B: - case IF_SVE_HW_4B_D: - case IF_SVE_HX_3A_E: - case IF_SVE_IJ_3A_D: - case IF_SVE_IJ_3A_E: - case IF_SVE_IJ_3A_F: - case IF_SVE_IK_4A_G: - case IF_SVE_IJ_3A_G: - case IF_SVE_IK_4A_I: - case IF_SVE_IH_3A_F: - case IF_SVE_II_4A_H: - case IF_SVE_IH_3A: - case IF_SVE_IH_3A_A: - case IF_SVE_II_4A: - case IF_SVE_II_4A_B: - case IF_SVE_IU_4A: - case IF_SVE_IU_4A_C: - case IF_SVE_IU_4B: - case IF_SVE_IU_4B_D: - case IF_SVE_IV_3A: - case IF_SVE_IG_4A_F: - case IF_SVE_IG_4A_G: - case IF_SVE_IJ_3A: - case IF_SVE_IK_4A: - case IF_SVE_IK_4A_F: - case IF_SVE_IK_4A_H: - case IF_SVE_IU_4A_A: - case IF_SVE_IU_4B_B: - case IF_SVE_HX_3A_B: - case IF_SVE_IG_4A: - case IF_SVE_IG_4A_D: - case IF_SVE_IG_4A_E: - case IF_SVE_IF_4A: - case IF_SVE_IF_4A_A: - case IF_SVE_IM_3A: - case IF_SVE_IN_4A: - case IF_SVE_IX_4A: - case IF_SVE_IO_3A: - case IF_SVE_IP_4A: - case IF_SVE_IQ_3A: - case IF_SVE_IR_4A: - case IF_SVE_IS_3A: - case IF_SVE_IT_4A: - case IF_SVE_GI_4A: - case IF_SVE_IC_3A_C: - case IF_SVE_IC_3A: - case IF_SVE_IC_3A_B: - case IF_SVE_IC_3A_A: - case IF_SVE_IL_3A_C: - case IF_SVE_IL_3A: - case IF_SVE_IL_3A_B: - case IF_SVE_IL_3A_A: - case IF_SVE_IW_4A: - return PREDICATE_ZERO; - - case IF_SVE_BV_2A_J: - case IF_SVE_CP_3A: - case IF_SVE_CQ_3A: - case IF_SVE_AM_2A: - case IF_SVE_AN_3A: - case IF_SVE_AO_3A: - case IF_SVE_HL_3A: - case IF_SVE_HM_2A: - case IF_SVE_AA_3A: - case IF_SVE_BU_2A: - case IF_SVE_BV_2B: - case IF_SVE_HS_3A: - case IF_SVE_HP_3A: - case IF_SVE_HP_3B: - case IF_SVE_AR_4A: - case IF_SVE_BV_2A_A: - case IF_SVE_AB_3A: - case IF_SVE_ET_3A: - case IF_SVE_HU_4A: - case IF_SVE_HL_3B: - case IF_SVE_AD_3A: - case IF_SVE_AB_3B: - case IF_SVE_AE_3A: - case IF_SVE_EU_3A: - case IF_SVE_GT_4A: - case IF_SVE_AP_3A: - case IF_SVE_HO_3A: - case IF_SVE_HO_3B: - case IF_SVE_HO_3C: - case IF_SVE_GQ_3A: - case IF_SVE_HU_4B: - case IF_SVE_AQ_3A: - case IF_SVE_CU_3A: - case IF_SVE_AC_3A: - case IF_SVE_ER_3A: - case IF_SVE_GR_3A: - case IF_SVE_ES_3A: - case IF_SVE_HR_3A: - case IF_SVE_EP_3A: - case IF_SVE_GP_3A: - case IF_SVE_EQ_3A: - case IF_SVE_HQ_3A: - case IF_SVE_AS_4A: - case IF_SVE_CT_3A: - case IF_SVE_HV_4A: - return PREDICATE_MERGE; - - case IF_SVE_CZ_4A_A: - case IF_SVE_CZ_4A_L: - case IF_SVE_CE_2A: - case IF_SVE_CE_2B: - case IF_SVE_CE_2C: - case IF_SVE_CE_2D: - case IF_SVE_CF_2A: - case IF_SVE_CF_2B: - case IF_SVE_CF_2C: - case IF_SVE_CF_2D: - case IF_SVE_CI_3A: - case IF_SVE_CJ_2A: - case IF_SVE_DE_1A: - case IF_SVE_DH_1A: - case IF_SVE_DJ_1A: - case IF_SVE_DM_2A: - case IF_SVE_DN_2A: - case IF_SVE_DO_2A: - case IF_SVE_DP_2A: - case IF_SVE_DR_1A: - case IF_SVE_DT_3A: - case IF_SVE_DU_3A: - case IF_SVE_CK_2A: - return PREDICATE_SIZED; - - case IF_SVE_DB_3A: - // Second register could be ZERO or MERGE so handled at source. - assert(regpos != 2); - return PREDICATE_SIZED; - - case IF_SVE_DL_2A: - case IF_SVE_DY_3A: - case IF_SVE_DZ_1A: - return PREDICATE_N_SIZED; - - // This is a special case as the second register could be ZERO or MERGE. - // / - // Therefore, by default return NONE due to ambiguity. - case IF_SVE_AH_3A: - // TODO: Handle these cases. - assert(false); - break; - - case IF_SVE_JD_4B: - case IF_SVE_JD_4C: - case IF_SVE_JI_3A_A: - case IF_SVE_JJ_4A: - case IF_SVE_JJ_4A_B: - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - case IF_SVE_JJ_4B: - case IF_SVE_JJ_4B_E: - case IF_SVE_JN_3B: - case IF_SVE_JN_3C: - case IF_SVE_JD_4A: - case IF_SVE_JN_3A: - case IF_SVE_JD_4C_A: - case IF_SVE_JJ_4B_C: - case IF_SVE_JL_3A: - case IF_SVE_JN_3C_D: - case IF_SVE_HY_3A: - case IF_SVE_HY_3A_A: - case IF_SVE_HY_3B: - case IF_SVE_HZ_2A_B: - case IF_SVE_IA_2A: - case IF_SVE_IB_3A: - case IF_SVE_JK_4A: - case IF_SVE_JK_4A_B: - case IF_SVE_JK_4B: - case IF_SVE_IZ_4A: - case IF_SVE_IZ_4A_A: - case IF_SVE_JB_4A: - case IF_SVE_JM_3A: - case IF_SVE_CM_3A: - case IF_SVE_CN_3A: - case IF_SVE_CO_3A: - case IF_SVE_JA_4A: - case IF_SVE_CR_3A: - case IF_SVE_CS_3A: - case IF_SVE_CV_3A: - case IF_SVE_CV_3B: - case IF_SVE_DW_2A: // [] - case IF_SVE_DW_2B: // [] - case IF_SVE_JC_4A: - case IF_SVE_JO_3A: - case IF_SVE_JE_3A: - case IF_SVE_JF_4A: - case IF_SVE_AK_3A: - case IF_SVE_HE_3A: - case IF_SVE_AF_3A: - case IF_SVE_AG_3A: - case IF_SVE_AI_3A: - case IF_SVE_AJ_3A: - case IF_SVE_AL_3A: - case IF_SVE_CL_3A: - case IF_SVE_GS_3A: - case IF_SVE_HJ_3A: - case IF_SVE_IY_4A: - return PREDICATE_NONE; - - case IF_SVE_CX_4A: - case IF_SVE_CX_4A_A: - case IF_SVE_CY_3A: - case IF_SVE_CY_3B: - case IF_SVE_GE_4A: - case IF_SVE_HT_4A: - assert((regpos == 1) || (regpos == 2)); - return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); - - case IF_SVE_CZ_4A: - case IF_SVE_DA_4A: - case IF_SVE_DB_3B: - case IF_SVE_DC_3A: - assert((regpos >= 1) && (regpos <= 4)); - return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); - - case IF_SVE_CZ_4A_K: - assert((regpos >= 1) && (regpos <= 3)); - return (regpos == 2 ? PREDICATE_MERGE : PREDICATE_SIZED); - - case IF_SVE_DD_2A: - case IF_SVE_DF_2A: - assert((regpos >= 1) && (regpos <= 3)); - return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); - - case IF_SVE_DG_2A: - return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); - - case IF_SVE_DI_2A: - return (regpos == 1 ? PREDICATE_NONE : PREDICATE_SIZED); - - case IF_SVE_DK_3A: - assert((regpos == 2) || (regpos == 3)); - return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); - - case IF_SVE_HI_3A: - assert((regpos == 1) || (regpos == 2)); - return ((regpos == 2) ? PREDICATE_ZERO : PREDICATE_SIZED); - - case IF_SVE_DV_4A: - assert((regpos >= 1) && (regpos <= 3)); - return ((regpos == 3) ? PREDICATE_SIZED : PREDICATE_NONE); - - case IF_SVE_ID_2A: - case IF_SVE_JG_2A: - return PREDICATE_NONE; + if (disp >= 0) + { + ins = INS_add; + imm = disp; + } + else + { + ins = INS_sub; + imm = -disp; + } - default: - break; + if (imm <= 0x0fff) + { + fmt = IF_DI_2A; // add reg1,reg2,#disp + } + else + { + regNumber rsvdReg = codeGen->rsGetRsvdReg(); + codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); + fmt = IF_DR_3A; // add reg1,reg2,rsvdReg + } } - - assert(!"Unexpected instruction format"); - return PREDICATE_NONE; -} - -/***************************************************************************** - * - * Returns true if the SVE instruction has a LSL addr. - * This is for formats that have [, , LSL #N], [{, , LSL #N}] - */ -/*static*/ bool emitter::insSveIsLslN(instruction ins, insFormat fmt) -{ - switch (fmt) + else { - case IF_SVE_JD_4A: - switch (ins) + ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate + imm = disp; + if (imm == 0) + { + fmt = IF_LS_2A; + } + else if ((imm < 0) || ((imm & mask) != 0)) + { + if ((imm >= -256) && (imm <= 255)) { - case INS_sve_st1h: - return true; - - default: - break; + fmt = IF_LS_2C; } - break; - - case IF_SVE_JD_4B: - switch (ins) + else { - case INS_sve_st1w: - return true; - - default: - break; + useRegForImm = true; } - break; - - case IF_SVE_HW_4B: - switch (ins) + } + else if (imm > 0) + { + if (((imm & mask) == 0) && ((imm >> scale) < 0x1000)) { - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - case INS_sve_ld1w: - case INS_sve_ldff1w: - return true; + imm >>= scale; // The immediate is scaled by the size of the ld/st - default: - break; + fmt = IF_LS_2B; } - break; - - case IF_SVE_IG_4A: - switch (ins) + else { - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - return true; - - default: - break; + useRegForImm = true; } - break; + } - case IF_SVE_IG_4A_F: - switch (ins) - { - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - return true; + if (useRegForImm) + { + regNumber rsvdReg = codeGen->rsGetRsvdReg(); + codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); + fmt = IF_LS_3A; + } + } - default: - break; - } - break; + assert(fmt != IF_NONE); - case IF_SVE_IG_4A_G: - switch (ins) - { - case INS_sve_ldff1h: - return true; + // Try to optimize a load/store with an alternative instruction. + if (isLdrStr && emitComp->opts.OptimizationEnabled() && + OptimizeLdrStr(ins, attr, reg1, reg2, imm, size, fmt, true, varx, offs DEBUG_ARG(useRegForImm))) + { + return; + } - default: - break; - } - break; + instrDesc* id = emitNewInstrCns(attr, imm); - case IF_SVE_II_4A: - case IF_SVE_II_4A_B: - switch (ins) - { - case INS_sve_ld1d: - return true; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); - default: - break; - } - break; + id->idReg1(reg1); + id->idReg2(reg2); + id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs); + id->idSetIsLclVar(); - case IF_SVE_II_4A_H: - switch (ins) - { - case INS_sve_ld1w: - return true; +#ifdef DEBUG + id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs; +#endif - default: - break; - } - break; + dispIns(id); + appendToCurIG(id); +} - case IF_SVE_IK_4A: - switch (ins) - { - case INS_sve_ld1sw: - return true; +/***************************************************************************** + * + * Add an instruction referencing two register and consecutive stack-based local variable slots. + */ +void emitter::emitIns_R_R_S_S( + instruction ins, emitAttr attr1, emitAttr attr2, regNumber reg1, regNumber reg2, int varx, int offs) +{ + assert((ins == INS_ldp) || (ins == INS_ldnp)); + assert(EA_8BYTE == EA_SIZE(attr1)); + assert(EA_8BYTE == EA_SIZE(attr2)); + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(offs >= 0); - default: - break; - } - break; + insFormat fmt = IF_LS_3B; + int disp = 0; + const unsigned scale = 3; - case IF_SVE_IK_4A_G: - switch (ins) - { - case INS_sve_ld1sh: - return true; + /* Figure out the variable's frame position */ + int base; + bool FPbased; - default: - break; - } - break; + base = emitComp->lvaFrameAddress(varx, &FPbased); + disp = base + offs; - case IF_SVE_IK_4A_I: - switch (ins) - { - case INS_sve_ld1h: - return true; + // TODO-ARM64-CQ: with compLocallocUsed, should we use REG_SAVED_LOCALLOC_SP instead? + regNumber reg3 = FPbased ? REG_FPBASE : REG_SPBASE; + reg3 = encodingSPtoZR(reg3); - default: - break; - } - break; + bool useRegForAdr = true; + ssize_t imm = disp; + ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate + if (imm == 0) + { + useRegForAdr = false; + } + else + { + if ((imm & mask) == 0) + { + ssize_t immShift = imm >> scale; // The immediate is scaled by the size of the ld/st - case IF_SVE_IN_4A: - switch (ins) + if ((immShift >= -64) && (immShift <= 63)) { - case INS_sve_ldnt1d: - case INS_sve_ldnt1h: - case INS_sve_ldnt1w: - return true; - - default: - break; + fmt = IF_LS_3C; + useRegForAdr = false; + imm = immShift; } - break; + } + } - case IF_SVE_IP_4A: - switch (ins) - { - case INS_sve_ld1roh: - case INS_sve_ld1row: - case INS_sve_ld1rod: - case INS_sve_ld1rqh: - case INS_sve_ld1rqw: - case INS_sve_ld1rqd: - return true; + if (useRegForAdr) + { + regNumber rsvd = codeGen->rsGetRsvdReg(); + emitIns_R_R_Imm(INS_add, EA_PTRSIZE, rsvd, reg3, imm); + reg3 = rsvd; + imm = 0; + } - default: - break; - } - break; + assert(fmt != IF_NONE); - case IF_SVE_IR_4A: - switch (ins) - { - case INS_sve_ld2q: - case INS_sve_ld3q: - case INS_sve_ld4q: - return true; + instrDesc* id = emitNewInstrCns(attr1, imm); - default: - break; - } - break; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); - case IF_SVE_IT_4A: - switch (ins) - { - case INS_sve_ld2h: - case INS_sve_ld2w: - case INS_sve_ld2d: - case INS_sve_ld3h: - case INS_sve_ld3w: - case INS_sve_ld3d: - case INS_sve_ld4h: - case INS_sve_ld4w: - case INS_sve_ld4d: - return true; + // Record the attribute for the second register in the pair + if (EA_IS_GCREF(attr2)) + { + id->idGCrefReg2(GCT_GCREF); + } + else if (EA_IS_BYREF(attr2)) + { + id->idGCrefReg2(GCT_BYREF); + } + else + { + id->idGCrefReg2(GCT_NONE); + } - default: - break; - } - break; + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs); + id->idSetIsLclVar(); - case IF_SVE_IU_4B: - switch (ins) - { - case INS_sve_ld1sw: - case INS_sve_ldff1sw: - case INS_sve_ld1d: - case INS_sve_ldff1d: - return true; +#ifdef DEBUG + id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs; +#endif - default: - break; - } - break; + dispIns(id); + appendToCurIG(id); +} - case IF_SVE_JB_4A: - switch (ins) - { - case INS_sve_stnt1h: - case INS_sve_stnt1w: - case INS_sve_stnt1d: - return true; +/***************************************************************************** + * + * Add an instruction referencing a stack-based local variable and a register + */ +void emitter::emitIns_S_R(instruction ins, emitAttr attr, regNumber reg1, int varx, int offs) +{ + assert(offs >= 0); + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + int disp = 0; + unsigned scale = 0; + bool isVectorStore = false; + bool isStr = false; - default: - break; - } + // TODO-ARM64-CQ: use unscaled loads? + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_strb: + scale = 0; + assert(isGeneralRegisterOrZR(reg1)); break; - case IF_SVE_JC_4A: - switch (ins) - { - case INS_sve_st2h: - case INS_sve_st2w: - case INS_sve_st2d: - case INS_sve_st3h: - case INS_sve_st3w: - case INS_sve_st3d: - case INS_sve_st4h: - case INS_sve_st4w: - case INS_sve_st4d: - return true; - - default: - break; - } + case INS_strh: + scale = 1; + assert(isGeneralRegisterOrZR(reg1)); break; - case IF_SVE_JD_4C: - switch (ins) + case INS_str: + if (isGeneralRegisterOrZR(reg1)) { - case INS_sve_st1w: - case INS_sve_st1d: - return true; - - default: - break; + assert(isValidGeneralDatasize(size)); + scale = (size == EA_8BYTE) ? 3 : 2; } - break; - - case IF_SVE_JD_4C_A: - switch (ins) + else { - case INS_sve_st1d: - return true; - - default: - break; + assert(isVectorRegister(reg1)); + assert(isValidVectorLSDatasize(size)); + scale = NaturalScale_helper(size); + isVectorStore = true; } + isStr = true; break; - case IF_SVE_JF_4A: - switch (ins) - { - case INS_sve_st2q: - case INS_sve_st3q: - case INS_sve_st4q: - return true; + default: + NYI("emitIns_S_R"); // FP locals? + return; - default: - break; - } - break; + } // end switch (ins) - case IF_SVE_JJ_4B: - switch (ins) - { - case INS_sve_st1h: - case INS_sve_st1w: - case INS_sve_st1d: - return true; + /* Figure out the variable's frame position */ + int base; + bool FPbased; - default: - break; - } - break; + base = emitComp->lvaFrameAddress(varx, &FPbased); + disp = base + offs; + assert(scale >= 0); + if (isVectorStore) + { + assert(scale <= 4); + } + else + { + assert(scale <= 3); + } - case IF_SVE_HY_3B: - case IF_SVE_IB_3A: - switch (ins) - { - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - return true; + // TODO-ARM64-CQ: with compLocallocUsed, should we use REG_SAVED_LOCALLOC_SP instead? + regNumber reg2 = FPbased ? REG_FPBASE : REG_SPBASE; + reg2 = encodingSPtoZR(reg2); - default: - break; - } - break; + bool useRegForImm = false; + ssize_t imm = disp; + ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate + if (imm == 0) + { + fmt = IF_LS_2A; + } + else if ((imm < 0) || ((imm & mask) != 0)) + { + if ((imm >= -256) && (imm <= 255)) + { + fmt = IF_LS_2C; + } + else + { + useRegForImm = true; + } + } + else if (imm > 0) + { + if (((imm & mask) == 0) && ((imm >> scale) < 0x1000)) + { + imm >>= scale; // The immediate is scaled by the size of the ld/st - default: - break; + fmt = IF_LS_2B; + } + else + { + useRegForImm = true; + } } - return false; -} + if (useRegForImm) + { + // The reserved register is not stored in idReg3() since that field overlaps with iiaLclVar. + // It is instead implicit when idSetIsLclVar() is set, with this encoding format. + regNumber rsvdReg = codeGen->rsGetRsvdReg(); + codeGen->instGen_Set_Reg_To_Imm(EA_PTRSIZE, rsvdReg, imm); + fmt = IF_LS_3A; + } -/***************************************************************************** - * - * Returns true if the SVE instruction has a addr. - * This is for formats that have [, .T, ], [, .T, #N] - */ -/*static*/ bool emitter::insSveIsModN(instruction ins, insFormat fmt) -{ - switch (fmt) + assert(fmt != IF_NONE); + + // Try to optimize a store with an alternative instruction. + if (isStr && emitComp->opts.OptimizationEnabled() && + OptimizeLdrStr(ins, attr, reg1, reg2, imm, size, fmt, true, varx, offs DEBUG_ARG(useRegForImm))) { - case IF_SVE_JJ_4A: - case IF_SVE_JJ_4A_B: - switch (ins) - { - case INS_sve_st1d: - case INS_sve_st1h: - case INS_sve_st1w: - return true; + return; + } - default: - break; - } - break; + instrDesc* id = emitNewInstrCns(attr, imm); - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - switch (ins) - { - case INS_sve_st1h: - case INS_sve_st1w: - return true; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); - default: - break; - } - break; + id->idReg1(reg1); + id->idReg2(reg2); + id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs); + id->idSetIsLclVar(); - case IF_SVE_JK_4A: - case IF_SVE_JK_4A_B: - switch (ins) - { - case INS_sve_st1b: - return true; +#ifdef DEBUG + id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs; +#endif - default: - break; - } - break; + dispIns(id); + appendToCurIG(id); +} - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - switch (ins) - { - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ld1sb: - case INS_sve_ld1sh: - case INS_sve_ld1w: - case INS_sve_ldff1b: - case INS_sve_ldff1h: - case INS_sve_ldff1sb: - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - return true; +/***************************************************************************** + * + * Add an instruction referencing consecutive stack-based local variable slots and two registers + */ +void emitter::emitIns_S_S_R_R( + instruction ins, emitAttr attr1, emitAttr attr2, regNumber reg1, regNumber reg2, int varx, int offs) +{ + assert((ins == INS_stp) || (ins == INS_stnp)); + assert(EA_8BYTE == EA_SIZE(attr1)); + assert(EA_8BYTE == EA_SIZE(attr2)); + assert(isGeneralRegisterOrZR(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(offs >= 0); - default: - break; - } - break; + insFormat fmt = IF_LS_3B; + int disp = 0; + const unsigned scale = 3; - case IF_SVE_HW_4A_B: - case IF_SVE_HW_4A_C: - switch (ins) - { - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ld1w: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - return true; + /* Figure out the variable's frame position */ + int base; + bool FPbased; - default: - break; - } - break; + base = emitComp->lvaFrameAddress(varx, &FPbased); + disp = base + offs; - case IF_SVE_IU_4A: - switch (ins) - { - case INS_sve_ld1d: - case INS_sve_ld1sw: - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - return true; + // TODO-ARM64-CQ: with compLocallocUsed, should we use REG_SAVED_LOCALLOC_SP instead? + regNumber reg3 = FPbased ? REG_FPBASE : REG_SPBASE; - default: - break; - } - break; + bool useRegForAdr = true; + ssize_t imm = disp; + ssize_t mask = (1 << scale) - 1; // the mask of low bits that must be zero to encode the immediate + if (imm == 0) + { + useRegForAdr = false; + } + else + { + if ((imm & mask) == 0) + { + ssize_t immShift = imm >> scale; // The immediate is scaled by the size of the ld/st - case IF_SVE_IU_4A_A: - switch (ins) + if ((immShift >= -64) && (immShift <= 63)) { - case INS_sve_ld1sw: - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - return true; - - default: - break; + fmt = IF_LS_3C; + useRegForAdr = false; + imm = immShift; } - break; + } + } - case IF_SVE_IU_4A_C: - switch (ins) - { - case INS_sve_ld1d: - return true; + if (useRegForAdr) + { + regNumber rsvd = codeGen->rsGetRsvdReg(); + emitIns_R_R_Imm(INS_add, EA_PTRSIZE, rsvd, reg3, imm); + reg3 = rsvd; + imm = 0; + } - default: - break; - } - break; + assert(fmt != IF_NONE); - case IF_SVE_HY_3A: - case IF_SVE_HY_3A_A: - switch (ins) - { - case INS_sve_prfb: - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - return true; + instrDesc* id = emitNewInstrCns(attr1, imm); - default: - break; - } - break; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); - default: - break; + // Record the attribute for the second register in the pair + if (EA_IS_GCREF(attr2)) + { + id->idGCrefReg2(GCT_GCREF); + } + else if (EA_IS_BYREF(attr2)) + { + id->idGCrefReg2(GCT_BYREF); + } + else + { + id->idGCrefReg2(GCT_NONE); } - return false; + reg3 = encodingSPtoZR(reg3); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + id->idAddr()->iiaLclVar.initLclVarAddr(varx, offs); + id->idSetIsLclVar(); + +#ifdef DEBUG + id->idDebugOnlyInfo()->idVarRefOffs = emitVarRefOffs; +#endif + + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns 0, 1, 2, 3 or 4 depending on the instruction and format. - * This is for formats that have [, .T, ], [, .T, #N], [, , LSL #N], - * [{, , LSL #N}] + * Add an instruction referencing stack-based local variable and an immediate */ +void emitter::emitIns_S_I(instruction ins, emitAttr attr, int varx, int offs, int val) +{ + NYI("emitIns_S_I"); +} -/*static*/ int emitter::insSveGetLslOrModN(instruction ins, insFormat fmt) +/***************************************************************************** + * + * Add an instruction with a register + static member operands. + * Constant is stored into JIT data which is adjacent to code. + * No relocation is needed. PC-relative offset will be encoded directly into instruction. + * + */ +void emitter::emitIns_R_C( + instruction ins, emitAttr attr, regNumber reg, regNumber addrReg, CORINFO_FIELD_HANDLE fldHnd, int offs) { - switch (fmt) - { - case IF_SVE_JD_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1h: - return 1; + assert(offs >= 0); + assert(instrDesc::fitsInSmallCns(offs)); - default: - break; - } + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + instrDescJmp* id = emitNewInstrJmp(); + + switch (ins) + { + case INS_adr: + // This is case to get address to the constant data. + fmt = IF_LARGEADR; + assert(isGeneralRegister(reg)); + assert(isValidGeneralDatasize(size)); break; - case IF_SVE_JD_4B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) + case INS_ldr: + fmt = IF_LARGELDC; + if (isVectorRegister(reg)) { - case INS_sve_st1w: - return 2; - - default: - break; + assert(isValidVectorLSDatasize(size)); + // For vector (float/double) register, we should have an integer address reg to + // compute long address which consists of page address and page offset. + // For integer constant, this is not needed since the dest reg can be used to + // compute address as well as contain the final contents. + assert(isGeneralRegister(reg) || (addrReg != REG_NA)); } - break; - - case IF_SVE_HW_4B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) + else { - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - return 1; - - case INS_sve_ld1w: - case INS_sve_ldff1w: - return 2; - - default: - break; + assert(isGeneralRegister(reg)); + assert(isValidGeneralDatasize(size)); } break; - case IF_SVE_JJ_4A: - case IF_SVE_JJ_4A_B: - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - case IF_SVE_JK_4A: - case IF_SVE_JK_4A_B: - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - case IF_SVE_HW_4A_B: - case IF_SVE_HW_4A_C: - case IF_SVE_IU_4A: - case IF_SVE_IU_4A_A: - case IF_SVE_IU_4A_C: - assert(!insSveIsLslN(ins, fmt)); - assert(insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - switch (fmt) - { - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - return 1; - - default: - break; - } - return 0; - - case INS_sve_ld1w: - case INS_sve_ldff1w: - case INS_sve_ld1sw: - case INS_sve_ldff1sw: - switch (fmt) - { - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - case IF_SVE_IU_4A: - return 2; + default: + unreached(); + } - default: - break; - } - return 0; + assert(fmt != IF_NONE); - case INS_sve_ld1d: - case INS_sve_ldff1d: - switch (fmt) - { - case IF_SVE_IU_4A: - return 3; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); + id->idSmallCns(offs); + id->idOpSize(size); + id->idAddr()->iiaFieldHnd = fldHnd; + id->idSetIsBound(); // We won't patch address since we will know the exact distance once JIT code and data are + // allocated together. - default: - break; - } - return 0; + id->idReg1(reg); // destination register that will get the constant value. + if (addrReg != REG_NA) + { + id->idReg2(addrReg); // integer register to compute long address (used for vector dest when we end up with long + // address) + } + id->idjShort = false; // Assume loading constant from long address - case INS_sve_st1h: - switch (fmt) - { - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - return 0; + // Keep it long if it's in cold code. + id->idjKeepLong = emitComp->fgIsBlockCold(emitComp->compCurBB); - default: - break; - } - return 1; +#ifdef DEBUG + if (emitComp->opts.compLongAddress) + id->idjKeepLong = 1; +#endif // DEBUG - case INS_sve_st1w: - switch (fmt) - { - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - return 0; + // If it's possible to be shortened, then put it in jump list + // to be revisited by emitJumpDistBind. + if (!id->idjKeepLong) + { + /* Record the jump's IG and offset within it */ + id->idjIG = emitCurIG; + id->idjOffs = emitCurIGsize; - default: - break; - } - return 2; + /* Append this jump to this IG's jump list */ + id->idjNext = emitCurIGjmpList; + emitCurIGjmpList = id; - case INS_sve_st1d: - if (fmt == IF_SVE_JJ_4A_B) - { - return 0; - } - return 3; +#if EMITTER_STATS + emitTotalIGjmps++; +#endif + } - default: - break; - } - return 0; + dispIns(id); + appendToCurIG(id); +} - case IF_SVE_IG_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ldff1sw: - return 2; +/***************************************************************************** + * + * Add an instruction with a static member + constant. + */ - case INS_sve_ldff1d: - return 3; +void emitter::emitIns_C_I(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE fldHnd, ssize_t offs, ssize_t val) +{ + NYI("emitIns_C_I"); +} - default: - break; - } - break; +/***************************************************************************** + * + * Add an instruction with a static member + register operands. + */ - case IF_SVE_IG_4A_F: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ldff1sh: - return 1; +void emitter::emitIns_C_R(instruction ins, emitAttr attr, CORINFO_FIELD_HANDLE fldHnd, regNumber reg, int offs) +{ + assert(!"emitIns_C_R not supported for RyuJIT backend"); +} - case INS_sve_ldff1w: - return 2; +void emitter::emitIns_R_AR(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, int offs) +{ + NYI("emitIns_R_AR"); +} - default: - break; - } - break; +// This generates code to populate the access for TLS on linux +void emitter::emitIns_Adrp_Ldr_Add(emitAttr attr, + regNumber reg1, + regNumber reg2, + ssize_t addr DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) +{ + assert(emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI)); + assert(TargetOS::IsUnix); + assert(EA_IS_RELOC(attr)); + assert(EA_IS_CNS_TLSGD_RELOC(attr)); - case IF_SVE_IG_4A_G: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ldff1h: - return 1; + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_DI_1E; + bool needAdd = false; + instrDescJmp* id = emitNewInstrJmp(); - default: - break; - } - break; + // adrp + id->idIns(INS_adrp); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); + id->idOpSize(size); + id->idAddr()->iiaAddr = (BYTE*)addr; + id->idReg1(reg1); + id->idSetIsDspReloc(); + id->idSetTlsGD(); - case IF_SVE_II_4A: - case IF_SVE_II_4A_B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1d: - return 3; +#ifdef DEBUG + id->idDebugOnlyInfo()->idMemCookie = targetHandle; + id->idDebugOnlyInfo()->idFlags = gtFlags; +#endif - default: - break; - } - break; + dispIns(id); + appendToCurIG(id); - case IF_SVE_II_4A_H: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1w: - return 2; + // ldr + emitIns_R_R_I(INS_ldr, attr, reg2, reg1, (ssize_t)addr); - default: - break; - } - break; + // add + fmt = IF_DI_2A; + instrDesc* addId = emitNewInstr(attr); + assert(id->idIsReloc()); - case IF_SVE_IK_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1sw: - return 2; + addId->idIns(INS_add); + addId->idInsFmt(fmt); + addId->idInsOpt(INS_OPTS_NONE); + addId->idOpSize(size); + addId->idAddr()->iiaAddr = (BYTE*)addr; + addId->idReg1(reg1); + addId->idReg2(reg1); + addId->idSetTlsGD(); - default: - break; - } - break; + dispIns(addId); + appendToCurIG(addId); +} - case IF_SVE_IK_4A_G: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1sh: - return 1; +// This computes address from the immediate which is relocatable. +void emitter::emitIns_R_AI(instruction ins, + emitAttr attr, + regNumber ireg, + ssize_t addr DEBUGARG(size_t targetHandle) DEBUGARG(GenTreeFlags gtFlags)) +{ + assert(EA_IS_RELOC(attr)); + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_DI_1E; + bool needAdd = false; + instrDescJmp* id = emitNewInstrJmp(); - default: - break; - } + switch (ins) + { + case INS_adrp: + // This computes page address. + // page offset is needed using add. + needAdd = true; + break; + case INS_adr: break; + default: + unreached(); + } - case IF_SVE_IK_4A_I: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1h: - return 1; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); + id->idOpSize(size); + id->idAddr()->iiaAddr = (BYTE*)addr; + id->idReg1(ireg); + id->idSetIsDspReloc(); +#ifdef DEBUG + id->idDebugOnlyInfo()->idMemCookie = targetHandle; + id->idDebugOnlyInfo()->idFlags = gtFlags; +#endif - default: - break; - } - break; + dispIns(id); + appendToCurIG(id); - case IF_SVE_IN_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ldnt1h: - return 1; - case INS_sve_ldnt1w: - return 2; - case INS_sve_ldnt1d: - return 3; + if (needAdd) + { + // add reg, reg, imm + ins = INS_add; + fmt = IF_DI_2A; + instrDesc* id = emitNewInstr(attr); + assert(id->idIsReloc()); - default: - break; - } - break; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(INS_OPTS_NONE); + id->idOpSize(size); + id->idAddr()->iiaAddr = (BYTE*)addr; + id->idReg1(ireg); + id->idReg2(ireg); - case IF_SVE_IP_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1roh: - case INS_sve_ld1rqh: - return 1; + dispIns(id); + appendToCurIG(id); + } +} - case INS_sve_ld1row: - case INS_sve_ld1rqw: - return 2; - case INS_sve_ld1rod: - case INS_sve_ld1rqd: - return 3; +void emitter::emitIns_AR_R(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, int offs) +{ + NYI("emitIns_AR_R"); +} - default: - break; - } - break; +void emitter::emitIns_R_ARR(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, regNumber rg2, int disp) +{ + NYI("emitIns_R_ARR"); +} - case IF_SVE_IR_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld2q: - case INS_sve_ld3q: - case INS_sve_ld4q: - return 4; +void emitter::emitIns_ARR_R(instruction ins, emitAttr attr, regNumber ireg, regNumber reg, regNumber rg2, int disp) +{ + NYI("emitIns_R_ARR"); +} - default: - break; - } - break; +void emitter::emitIns_R_ARX( + instruction ins, emitAttr attr, regNumber ireg, regNumber reg, regNumber rg2, unsigned mul, int disp) +{ + NYI("emitIns_R_ARR"); +} - case IF_SVE_IT_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld2h: - case INS_sve_ld3h: - case INS_sve_ld4h: - return 1; +/***************************************************************************** + * + * Record that a jump instruction uses the short encoding + * + */ +void emitter::emitSetShortJump(instrDescJmp* id) +{ + if (id->idjKeepLong) + return; - case INS_sve_ld2w: - case INS_sve_ld3w: - case INS_sve_ld4w: - return 2; + insFormat fmt = IF_NONE; + if (emitIsCondJump(id)) + { + switch (id->idIns()) + { + case INS_cbz: + case INS_cbnz: + fmt = IF_BI_1A; + break; + case INS_tbz: + case INS_tbnz: + fmt = IF_BI_1B; + break; + default: + fmt = IF_BI_0B; + break; + } + } + else if (emitIsLoadLabel(id)) + { + fmt = IF_DI_1E; + } + else if (emitIsLoadConstant(id)) + { + fmt = IF_LS_1A; + } + else + { + unreached(); + } - case INS_sve_ld2d: - case INS_sve_ld3d: - case INS_sve_ld4d: - return 3; + id->idInsFmt(fmt); + id->idjShort = true; +} - default: - break; - } - break; +/***************************************************************************** + * + * Add a label instruction. + */ - case IF_SVE_IU_4B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1sw: - case INS_sve_ldff1sw: - return 2; +void emitter::emitIns_R_L(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg) +{ + assert(dst->HasFlag(BBF_HAS_LABEL)); - case INS_sve_ld1d: - case INS_sve_ldff1d: - return 3; + insFormat fmt = IF_NONE; - default: - break; - } + switch (ins) + { + case INS_adr: + fmt = IF_LARGEADR; break; + default: + unreached(); + } - case IF_SVE_JB_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_stnt1h: - return 1; - - case INS_sve_stnt1w: - return 2; + instrDescJmp* id = emitNewInstrJmp(); - case INS_sve_stnt1d: - return 3; + id->idIns(ins); + id->idInsFmt(fmt); + id->idjShort = false; + id->idAddr()->iiaBBlabel = dst; + id->idReg1(reg); + id->idOpSize(EA_PTRSIZE); - default: - break; - } - break; +#ifdef DEBUG + // Mark the catch return + if (emitComp->compCurBB->KindIs(BBJ_EHCATCHRET)) + { + id->idDebugOnlyInfo()->idCatchRet = true; + } +#endif // DEBUG - case IF_SVE_JC_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st2h: - case INS_sve_st3h: - case INS_sve_st4h: - return 1; + id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); - case INS_sve_st2w: - case INS_sve_st3w: - case INS_sve_st4w: - return 2; +#ifdef DEBUG + if (emitComp->opts.compLongAddress) + id->idjKeepLong = 1; +#endif // DEBUG - case INS_sve_st2d: - case INS_sve_st3d: - case INS_sve_st4d: - return 3; + /* Record the jump's IG and offset within it */ - default: - break; - } - break; + id->idjIG = emitCurIG; + id->idjOffs = emitCurIGsize; - case IF_SVE_JD_4C: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1w: - return 2; + /* Append this jump to this IG's jump list */ - case INS_sve_st1d: - return 3; + id->idjNext = emitCurIGjmpList; + emitCurIGjmpList = id; - default: - break; - } - break; +#if EMITTER_STATS + emitTotalIGjmps++; +#endif - case IF_SVE_JD_4C_A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1d: - return 3; + dispIns(id); + appendToCurIG(id); +} - default: - break; - } - break; +/***************************************************************************** + * + * Add a data label instruction. + */ - case IF_SVE_JF_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st2q: - case INS_sve_st3q: - case INS_sve_st4q: - return 4; +void emitter::emitIns_R_D(instruction ins, emitAttr attr, unsigned offs, regNumber reg) +{ + NYI("emitIns_R_D"); +} - default: - break; - } - break; +void emitter::emitIns_J_R(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg) +{ + assert((ins == INS_cbz) || (ins == INS_cbnz)); - case IF_SVE_JJ_4B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1h: - return 1; + assert(dst != nullptr); + assert(dst->HasFlag(BBF_HAS_LABEL)); - case INS_sve_st1w: - return 2; + insFormat fmt = IF_LARGEJMP; - case INS_sve_st1d: - return 3; + instrDescJmp* id = emitNewInstrJmp(); - default: - break; - } - break; + id->idIns(ins); + id->idInsFmt(fmt); + id->idReg1(reg); + id->idjShort = false; + id->idOpSize(EA_SIZE(attr)); - case IF_SVE_HY_3A: - case IF_SVE_HY_3A_A: - assert(!insSveIsLslN(ins, fmt)); - assert(insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_prfb: - return 0; + id->idAddr()->iiaBBlabel = dst; + id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); - case INS_sve_prfh: - return 1; + /* Record the jump's IG and offset within it */ - case INS_sve_prfw: - return 2; + id->idjIG = emitCurIG; + id->idjOffs = emitCurIGsize; - case INS_sve_prfd: - return 3; + /* Append this jump to this IG's jump list */ - default: - break; - } - break; + id->idjNext = emitCurIGjmpList; + emitCurIGjmpList = id; - case IF_SVE_HY_3B: - case IF_SVE_IB_3A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_prfh: - return 1; +#if EMITTER_STATS + emitTotalIGjmps++; +#endif - case INS_sve_prfw: - return 2; + dispIns(id); + appendToCurIG(id); +} - case INS_sve_prfd: - return 3; +void emitter::emitIns_J_R_I(instruction ins, emitAttr attr, BasicBlock* dst, regNumber reg, int imm) +{ + assert((ins == INS_tbz) || (ins == INS_tbnz)); - default: - break; - } - break; + assert(dst != nullptr); + assert(dst->HasFlag(BBF_HAS_LABEL)); + assert((EA_SIZE(attr) == EA_4BYTE) || (EA_SIZE(attr) == EA_8BYTE)); + assert(imm < ((EA_SIZE(attr) == EA_4BYTE) ? 32 : 64)); - default: - break; - } + insFormat fmt = IF_LARGEJMP; - assert(!"Unexpected instruction format"); - return 0; -} + instrDescJmp* id = emitNewInstrJmp(); -/***************************************************************************** - * - * Returns true if the specified instruction can encode the 'dtype' field. - */ + id->idIns(ins); + id->idInsFmt(fmt); + id->idReg1(reg); + id->idjShort = false; + id->idSmallCns(imm); + id->idOpSize(EA_SIZE(attr)); -/*static*/ bool emitter::canEncodeSveElemsize_dtype(instruction ins) -{ - switch (ins) - { - case INS_sve_ld1w: - case INS_sve_ld1sb: - case INS_sve_ld1b: - case INS_sve_ld1sh: - case INS_sve_ld1h: - case INS_sve_ldnf1sh: - case INS_sve_ldnf1w: - case INS_sve_ldnf1h: - case INS_sve_ldnf1sb: - case INS_sve_ldnf1b: - case INS_sve_ldff1b: - case INS_sve_ldff1sb: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - return true; + id->idAddr()->iiaBBlabel = dst; + id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); - default: - return false; - } -} + /* Record the jump's IG and offset within it */ -/***************************************************************************** - * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * for the 'dtype' field. - */ + id->idjIG = emitCurIG; + id->idjOffs = emitCurIGsize; -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(instruction ins, emitAttr size, code_t code) -{ - assert(canEncodeSveElemsize_dtype(ins)); - assert(ins != INS_sve_ld1w); - switch (size) - { - case EA_1BYTE: - switch (ins) - { - case INS_sve_ld1b: - case INS_sve_ldnf1b: - case INS_sve_ldff1b: - return code; // By default, the instruction already encodes 8-bit. + /* Append this jump to this IG's jump list */ - default: - assert(!"Invalid instruction for encoding dtype."); - } - return code; + id->idjNext = emitCurIGjmpList; + emitCurIGjmpList = id; - case EA_2BYTE: - switch (ins) - { - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ldnf1b: - case INS_sve_ldnf1h: - case INS_sve_ldff1b: - case INS_sve_ldff1h: - return code | (1 << 21); // Set bit '21' to 1. +#if EMITTER_STATS + emitTotalIGjmps++; +#endif - case INS_sve_ld1sb: - case INS_sve_ldnf1sb: - case INS_sve_ldff1sb: - return code | (1 << 22); // Set bit '22' to 1. + dispIns(id); + appendToCurIG(id); +} - default: - assert(!"Invalid instruction for encoding dtype."); - } - return code; +void emitter::emitIns_J(instruction ins, BasicBlock* dst, int instrCount) +{ + insFormat fmt = IF_NONE; - case EA_4BYTE: - switch (ins) - { - case INS_sve_ldnf1w: - case INS_sve_ldff1w: - return code; // By default, the instruction already encodes 32-bit. - - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ldnf1b: - case INS_sve_ldnf1h: - case INS_sve_ldff1b: - case INS_sve_ldff1h: - return code | (1 << 22); // Set bit '22' to 1. - - case INS_sve_ld1sb: - case INS_sve_ld1sh: - case INS_sve_ldnf1sb: - case INS_sve_ldnf1sh: - case INS_sve_ldff1sb: - case INS_sve_ldff1sh: - return code | (1 << 21); // Set bit '21' to 1. + if (dst != nullptr) + { + assert(dst->HasFlag(BBF_HAS_LABEL)); + } + else + { + assert(instrCount != 0); + } - default: - assert(!"Invalid instruction for encoding dtype."); - } - return code; + /* Figure out the encoding format of the instruction */ - case EA_8BYTE: - switch (ins) - { - case INS_sve_ldnf1w: - case INS_sve_ldff1w: - return code | (1 << 21); // Set bit '21' to 1. Set bit '15' to 1. - - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ldnf1b: - case INS_sve_ldnf1h: - case INS_sve_ldff1b: - case INS_sve_ldff1h: - return (code | (1 << 22)) | (1 << 21); // Set bit '22' and '21' to 1. - - case INS_sve_ld1sb: - case INS_sve_ld1sh: - case INS_sve_ldnf1sb: - case INS_sve_ldnf1sh: - case INS_sve_ldff1sb: - case INS_sve_ldff1sh: - return code; // By default, the instruction already encodes 64-bit. + switch (ins) + { + case INS_bl_local: + case INS_b: + // Unconditional jump is a single form. + // Assume is long in case we cross hot/cold sections. + fmt = IF_BI_0A; + break; - default: - assert(!"Invalid instruction for encoding dtype."); - } - return code; + case INS_beq: + case INS_bne: + case INS_bhs: + case INS_blo: + case INS_bmi: + case INS_bpl: + case INS_bvs: + case INS_bvc: + case INS_bhi: + case INS_bls: + case INS_bge: + case INS_blt: + case INS_bgt: + case INS_ble: + // Assume conditional jump is long. + fmt = IF_LARGEJMP; + break; default: - assert(!"Invalid size for encoding dtype."); + unreached(); + break; } - return code; -} + instrDescJmp* id = emitNewInstrJmp(); -/***************************************************************************** - * - * Returns the encoding to select the 4/8/16 byte elemsize for the Arm64 Sve vector instruction 'ld1w' - * for the 'dtype' field. - */ + id->idIns(ins); + id->idInsFmt(fmt); + id->idjShort = false; -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype_ld1w(instruction ins, - insFormat fmt, - emitAttr size, - code_t code) -{ - assert(canEncodeSveElemsize_dtype(ins)); - assert(ins == INS_sve_ld1w); - switch (size) +#ifdef DEBUG + // Mark the finally call + if (ins == INS_bl_local && emitComp->compCurBB->KindIs(BBJ_CALLFINALLY)) { - case EA_4BYTE: - switch (fmt) - { - case IF_SVE_IH_3A_F: - // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for S. - return (code | (1 << 15)) | (1 << 22); // Set bit '22' and '15' to 1. - - case IF_SVE_II_4A_H: - // Note: Bit '14' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for S. - return (code | (1 << 14)) | (1 << 22); // Set bit '22' and '14' to 1. + id->idDebugOnlyInfo()->idFinallyCall = true; + } +#endif // DEBUG - default: - break; - } - break; + if (dst != nullptr) + { + id->idAddr()->iiaBBlabel = dst; - case EA_8BYTE: - switch (fmt) - { - case IF_SVE_IH_3A_F: - // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for D. - return ((code | (1 << 15)) | (1 << 22)) | (1 << 21); // Set bit '22', '21' and '15' to 1. + // Skip unconditional jump that has a single form. + // The target needs to be relocated. + id->idjKeepLong = emitComp->fgInDifferentRegions(emitComp->compCurBB, dst); - case IF_SVE_II_4A_H: - // Note: Bit '14' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for D. - return ((code | (1 << 14)) | (1 << 22)) | (1 << 21); // Set bit '22', '21' and '14' to 1. +#ifdef DEBUG + if (emitComp->opts.compLongAddress) // Force long branches + { + id->idjKeepLong = true; + } +#endif // DEBUG + } + else + { + id->idAddr()->iiaSetInstrCount(instrCount); + id->idjKeepLong = false; + /* This jump must be short */ + emitSetShortJump(id); + id->idSetIsBound(); + } - default: - break; - } - break; + /* Record the jump's IG and offset within it */ - case EA_16BYTE: - switch (fmt) - { - case IF_SVE_IH_3A_F: - return code | (1 << 20); // Set bit '20' to 1. + id->idjIG = emitCurIG; + id->idjOffs = emitCurIGsize; - case IF_SVE_II_4A_H: - // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for Q. - return code | (1 << 15); // Set bit '15' to 1. + /* Append this jump to this IG's jump list */ - default: - break; - } - break; + id->idjNext = emitCurIGjmpList; + emitCurIGjmpList = id; - default: - assert(!"Invalid size for encoding dtype."); - break; - } +#if EMITTER_STATS + emitTotalIGjmps++; +#endif - assert(!"Invalid instruction format"); - return code; + dispIns(id); + appendToCurIG(id); } /***************************************************************************** * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * for the 'dtypeh' and 'dtypel' fields. + * Add a call instruction (direct or indirect). + * argSize<0 means that the caller will pop the arguments + * + * The other arguments are interpreted depending on callType as shown: + * Unless otherwise specified, ireg,xreg,xmul,disp should have default values. + * + * EC_FUNC_TOKEN : addr is the method address + * EC_FUNC_ADDR : addr is the absolute address of the function + * + * If callType is one of these emitCallTypes, addr has to be NULL. + * EC_INDIR_R : "call ireg". + * + * For ARM xreg, xmul and disp are never used and should always be 0/REG_NA. + * + * Please consult the "debugger team notification" comment in genFnProlog(). */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtypeh_dtypel(instruction ins, - insFormat fmt, - emitAttr size, - code_t code) +void emitter::emitIns_Call(EmitCallType callType, + CORINFO_METHOD_HANDLE methHnd, + INDEBUG_LDISASM_COMMA(CORINFO_SIG_INFO* sigInfo) // used to report call sites to the EE + void* addr, + ssize_t argSize, + emitAttr retSize, + emitAttr secondRetSize, + VARSET_VALARG_TP ptrVars, + regMaskTP gcrefRegs, + regMaskTP byrefRegs, + const DebugInfo& di /* = DebugInfo() */, + regNumber ireg /* = REG_NA */, + regNumber xreg /* = REG_NA */, + unsigned xmul /* = 0 */, + ssize_t disp /* = 0 */, + bool isJump /* = false */) { - switch (fmt) - { - case IF_SVE_IC_3A_A: - switch (size) - { - case EA_4BYTE: - switch (ins) - { - case INS_sve_ld1rsh: - return code | (1 << 13); // set bit '13' + /* Sanity check the arguments depending on callType */ - case INS_sve_ld1rw: - return code | (1 << 14); // set bit '14' + assert(callType < EC_COUNT); + assert((callType != EC_FUNC_TOKEN) || (addr != nullptr && ireg == REG_NA)); + assert(callType != EC_INDIR_R || (addr == nullptr && ireg < REG_COUNT)); - default: - break; - } - break; + // ARM never uses these + assert(xreg == REG_NA && xmul == 0 && disp == 0); - case EA_8BYTE: - switch (ins) - { - case INS_sve_ld1rsh: - return code; + // Our stack level should be always greater than the bytes of arguments we push. Just + // a sanity test. + assert((unsigned)abs(argSize) <= codeGen->genStackLevel); - case INS_sve_ld1rw: - return code | (1 << 14) | (1 << 13); // set bits '14' and '13' + // Trim out any callee-trashed registers from the live set. + regMaskTP savedSet = emitGetGCRegsSavedOrModified(methHnd); + gcrefRegs &= savedSet; + byrefRegs &= savedSet; - default: - break; - } - break; +#ifdef DEBUG + if (EMIT_GC_VERBOSE) + { + printf("Call: GCvars=%s ", VarSetOps::ToString(emitComp, ptrVars)); + dumpConvertedVarSet(emitComp, ptrVars); + printf(", gcrefRegs="); + printRegMaskInt(gcrefRegs); + emitDispRegSet(gcrefRegs); + printf(", byrefRegs="); + printRegMaskInt(byrefRegs); + emitDispRegSet(byrefRegs); + printf("\n"); + } +#endif - default: - break; - } - break; + /* Managed RetVal: emit sequence point for the call */ + if (emitComp->opts.compDbgInfo && di.GetLocation().IsValid()) + { + codeGen->genIPmappingAdd(IPmappingDscKind::Normal, di, false); + } - case IF_SVE_IC_3A_B: - switch (size) - { - case EA_2BYTE: - switch (ins) - { - case INS_sve_ld1rh: - return code | (1 << 13); // set bit '13' + /* + We need to allocate the appropriate instruction descriptor based + on whether this is a direct/indirect call, and whether we need to + record an updated set of live GC variables. + */ + instrDesc* id; - case INS_sve_ld1rsb: - return code | (1 << 24) | (1 << 14); // set bit '24' and '14' + assert(argSize % REGSIZE_BYTES == 0); + int argCnt = (int)(argSize / (int)REGSIZE_BYTES); - default: - break; - } - break; + if (callType == EC_INDIR_R) + { + /* Indirect call, virtual calls */ - case EA_4BYTE: - switch (ins) - { - case INS_sve_ld1rh: - return code | (1 << 14); // set bit '14' + id = emitNewInstrCallInd(argCnt, 0 /* disp */, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize); + } + else + { + /* Helper/static/nonvirtual/function calls (direct or through handle), + and calls to an absolute addr. */ - case INS_sve_ld1rsb: - return code | (1 << 24) | (1 << 13); // set bit '24' and '13' + assert(callType == EC_FUNC_TOKEN); - default: - break; - } - break; + id = emitNewInstrCallDir(argCnt, ptrVars, gcrefRegs, byrefRegs, retSize, secondRetSize); + } - case EA_8BYTE: - switch (ins) - { - case INS_sve_ld1rh: - return code | (1 << 14) | (1 << 13); // set bits '14' and '13' + /* Update the emitter's live GC ref sets */ - case INS_sve_ld1rsb: - return code | (1 << 24); // set bit '24' + VarSetOps::Assign(emitComp, emitThisGCrefVars, ptrVars); + emitThisGCrefRegs = gcrefRegs; + emitThisByrefRegs = byrefRegs; - default: - break; - } - break; + id->idSetIsNoGC(emitNoGChelper(methHnd)); - default: - break; - } - break; + /* Set the instruction - special case jumping a function */ + instruction ins; + insFormat fmt = IF_NONE; - case IF_SVE_IC_3A_C: - assert(ins == INS_sve_ld1rb); - switch (size) - { - case EA_1BYTE: - return code; + /* Record the address: method, indirection, or funcptr */ - case EA_2BYTE: - return code | (1 << 13); // set bit '13' + if (callType == EC_INDIR_R) + { + /* This is an indirect call (either a virtual call or func ptr call) */ - case EA_4BYTE: - return code | (1 << 14); // set bit '14' + if (isJump) + { + ins = INS_br_tail; // INS_br_tail Reg + } + else + { + ins = INS_blr; // INS_blr Reg + } + fmt = IF_BR_1B; + + id->idIns(ins); + id->idInsFmt(fmt); + + assert(xreg == REG_NA); + if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && EA_IS_CNS_TLSGD_RELOC(retSize)) + { + // For NativeAOT linux/arm64, we need to also record the relocation of methHnd. + // Since we do not have space to embed it in instrDesc, we store the register in + // reg1 and instead use the `iiaAdd` to store the method handle. Likewise, during + // emitOutputInstr, we retrieve the register from reg1 for this specific case. + id->idSetTlsGD(); + id->idReg1(ireg); + id->idAddr()->iiaAddr = (BYTE*)methHnd; + } + else + { + id->idReg3(ireg); + } + } + else + { + /* This is a simple direct call: "call helper/method/addr" */ + + assert(callType == EC_FUNC_TOKEN); - case EA_8BYTE: - return code | (1 << 14) | (1 << 13); // set bits '14' and '13' + assert(addr != NULL); - default: - break; - } - break; + if (isJump) + { + ins = INS_b_tail; // INS_b_tail imm28 + } + else + { + ins = INS_bl; // INS_bl imm28 + } + fmt = IF_BI_0C; - default: - break; - } + id->idIns(ins); + id->idInsFmt(fmt); - assert(!"Unexpected instruction format"); - return code; -} + id->idAddr()->iiaAddr = (BYTE*)addr; -/***************************************************************************** - * - * Returns the encoding for the immediate value as 9-bits at bit locations '21-16' for high and '12-10' for low. - */ + if (emitComp->opts.compReloc) + { + id->idSetIsDspReloc(); + } + } -/*static*/ emitter::code_t emitter::insEncodeSimm9h9l_21_to_16_and_12_to_10(ssize_t imm) -{ - assert(isValidSimm<9>(imm)); +#ifdef DEBUG + if (EMIT_GC_VERBOSE) + { + if (id->idIsLargeCall()) + { + printf("[%02u] Rec call GC vars = %s\n", id->idDebugOnlyInfo()->idNum, + VarSetOps::ToString(emitComp, ((instrDescCGCA*)id)->idcGCvars)); + } + } +#endif - if (imm < 0) + if (m_debugInfoSize > 0) { - imm = (imm & 0x1FF); + INDEBUG(id->idDebugOnlyInfo()->idCallSig = sigInfo); + id->idDebugOnlyInfo()->idMemCookie = (size_t)methHnd; // method token } - code_t h = (code_t)(imm & 0x1F8) << 13; // encode high 6-bits at locations '21-16' - code_t l = (code_t)((imm & ~0x1F8) & 0x7) << 10; // encode low 3-bits at locations '12-10' +#ifdef LATE_DISASM + if (addr != nullptr) + { + codeGen->getDisAssembler().disSetMethod((size_t)addr, methHnd); + } +#endif // LATE_DISASM - return (h | l); + dispIns(id); + appendToCurIG(id); + emitLastMemBarrier = nullptr; // Cannot optimize away future memory barriers } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 2 as 4-bits at bit locations '19-16'. + * Returns true if 'imm' is valid Cond encoding */ -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf2_19_to_16(ssize_t imm) +/*static*/ bool emitter::isValidImmCond(ssize_t imm) { - assert((isValidSimm_MultipleOf<4, 2>(imm))); - return insEncodeSimm<19, 16>(imm / 2); -} + // range check the ssize_t value, to make sure it is a small unsigned value + // and that only the bits in the cfi.cond are set + if ((imm < 0) || (imm > 0xF)) + return false; -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 3 as 4-bits at bit locations '19-16'. - */ + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf3_19_to_16(ssize_t imm) -{ - assert((isValidSimm_MultipleOf<4, 3>(imm))); - return insEncodeSimm<19, 16>(imm / 3); + return (cfi.cond <= INS_COND_LE); // Don't allow 14 & 15 (AL & NV). } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 4 as 4-bits at bit locations '19-16'. + * Returns true if 'imm' is valid Cond/Flags encoding */ -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf4_19_to_16(ssize_t imm) +/*static*/ bool emitter::isValidImmCondFlags(ssize_t imm) { - assert((isValidSimm_MultipleOf<4, 4>(imm))); - return insEncodeSimm<19, 16>(imm / 4); -} + // range check the ssize_t value, to make sure it is a small unsigned value + // and that only the bits in the cfi.cond or cfi.flags are set + if ((imm < 0) || (imm > 0xFF)) + return false; -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 16 as 4-bits at bit locations '19-16'. - */ + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf16_19_to_16(ssize_t imm) -{ - assert((isValidSimm_MultipleOf<4, 16>(imm))); - return insEncodeSimm<19, 16>(imm / 16); + return (cfi.cond <= INS_COND_LE); // Don't allow 14 & 15 (AL & NV). } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 32 as 4-bits at bit locations '19-16'. + * Returns true if 'imm' is valid Cond/Flags/Imm5 encoding */ -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf32_19_to_16(ssize_t imm) +/*static*/ bool emitter::isValidImmCondFlagsImm5(ssize_t imm) { - assert((isValidSimm_MultipleOf<4, 32>(imm))); - return insEncodeSimm<19, 16>(imm / 32); + // range check the ssize_t value, to make sure it is a small unsigned value + // and that only the bits in the cfi.cond, cfi.flags or cfi.imm5 are set + if ((imm < 0) || (imm > 0x1FFF)) + return false; + + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + + return (cfi.cond <= INS_COND_LE); // Don't allow 14 & 15 (AL & NV). } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 2 as 5-bits at bit locations '20-16'. + * Returns an encoding for the specified register used in the 'Rd' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf2_20_to_16(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Rd(regNumber reg) { - assert((isValidUimm_MultipleOf<5, 2>(imm))); - return insEncodeUimm<20, 16>(imm / 2); + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 0) && (ureg <= 31)); + return ureg; } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 4 as 5-bits at bit locations '20-16'. + * Returns an encoding for the specified register used in the 'Rt' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf4_20_to_16(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Rt(regNumber reg) { - assert((isValidUimm_MultipleOf<5, 4>(imm))); - return insEncodeUimm<20, 16>(imm / 4); + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 0) && (ureg <= 31)); + return ureg; } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 8 as 5-bits at bit locations '20-16'. + * Returns an encoding for the specified register used in the 'Rn' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf8_20_to_16(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Rn(regNumber reg) { - assert((isValidUimm_MultipleOf<5, 8>(imm))); - return insEncodeUimm<20, 16>(imm / 8); + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 5; } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 2 as 6-bits at bit locations '21-16'. + * Returns an encoding for the specified register used in the 'Rm' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf2_21_to_16(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Rm(regNumber reg) { - assert((isValidUimm_MultipleOf<6, 2>(imm))); - return insEncodeUimm<21, 16>(imm / 2); + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 16; } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 4 as 6-bits at bit locations '21-16'. + * Returns an encoding for the specified register used in the 'Ra' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf4_21_to_16(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Ra(regNumber reg) { - assert((isValidUimm_MultipleOf<6, 4>(imm))); - return insEncodeUimm<21, 16>(imm / 4); + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 10; } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 8 as 6-bits at bit locations '21-16'. + * Returns an encoding for the specified register used in the 'Vd' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf8_21_to_16(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Vd(regNumber reg) { - assert((isValidUimm_MultipleOf<6, 8>(imm))); - return insEncodeUimm<21, 16>(imm / 8); + assert(emitter::isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 31)); + return ureg; } /***************************************************************************** * - * Returns the encoding for the immediate value as 1-bit at bit locations '23'. + * Returns an encoding for the specified register used in the 'Vt' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm1_23(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Vt(regNumber reg) { - assert(isValidUimm<1>(imm)); - return (code_t)imm << 23; + assert(emitter::isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 31)); + return ureg; } /***************************************************************************** * - * Returns the encoding for the immediate value as 3-bits at bit locations '23-22' for high and '12' for low. + * Returns an encoding for the specified register used in the 'Vn' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm3h3l_23_to_22_and_12(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Vn(regNumber reg) { - assert(isValidUimm<3>(imm)); - - code_t h = (code_t)(imm & 0x6) << 21; // encode high 2-bits at locations '23-22' - code_t l = (code_t)(imm & 0x1) << 12; // encode low 1-bit at locations '12' - - return (h | l); + assert(emitter::isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 5; } /***************************************************************************** * - * Returns the encoding for the immediate value as 4-bits starting from 1, at bit locations '19-16'. + * Returns an encoding for the specified register used in the 'Vm' position */ -/*static*/ emitter::code_t emitter::insEncodeUimm4From1_19_to_16(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Vm(regNumber reg) { - assert(isValidUimmFrom1<4>(imm)); - return (code_t)(imm - 1) << 16; + assert(emitter::isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 16; } /***************************************************************************** * - * Returns the encoding for the immediate value as 8-bits at bit locations '12-5'. + * Returns an encoding for the specified register used in the 'Va' position */ -/*static*/ emitter::code_t emitter::insEncodeImm8_12_to_5(ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeReg_Va(regNumber reg) { - assert(isValidSimm<8>(imm) || isValidUimm<8>(imm)); - return (code_t)((imm & 0xFF) << 5); + assert(emitter::isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 10; } /***************************************************************************** * - * Returns the encoding to select the 4/8-byte width specifier - * at bit location 22 for an Arm64 Sve instruction. + * Return an encoding for the specified 'V' register used in '4' thru '0' position. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_R_22(emitAttr size) -{ - if (size == EA_8BYTE) - { - return 0x400000; // set the bit at location 22 - } - assert(size == EA_4BYTE); - return 0; +/*static*/ emitter::code_t emitter::insEncodeReg_V_4_to_0(regNumber reg) +{ + assert(isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 0; } /***************************************************************************** * - * Returns the immediate value for instructions that encode it as a difference - * from tszh:tszl:imm3. + * Return an encoding for the specified 'V' register used in '9' thru '5' position. */ -/*static*/ ssize_t emitter::insGetImmDiff(const ssize_t imm, const insOpts opt) -{ - switch (opt) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); - return (8 - imm); - - case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); - return (16 - imm); - - case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); - return (32 - imm); - case INS_OPTS_SCALABLE_D: - assert(isValidUimmFrom1<6>(imm)); - return (64 - imm); - - default: - unreached(); - break; - } - - return 0; +/*static*/ emitter::code_t emitter::insEncodeReg_V_9_to_5(regNumber reg) +{ + assert(isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 5; } /***************************************************************************** * - * Returns the two 5-bit signed immediates encoded in the following format: - * njjj jjmi iiii - * - iiiii: the absolute value of imm1 - * - m: 1 if imm1 is negative, 0 otherwise - * - jjjjj: the absolute value of imm2 - * - n: 1 if imm2 is negative, 0 otherwise + * Return an encoding for the specified 'V' register used in '20' thru '16' position. */ -/*static*/ ssize_t emitter::insEncodeTwoSimm5(ssize_t imm1, ssize_t imm2) -{ - assert(isValidSimm<5>(imm1)); - assert(isValidSimm<5>(imm2)); - ssize_t immOut = 0; - if (imm1 < 0) - { - // Set bit location 5 to indicate imm1 is negative - immOut |= 0x20; - imm1 *= -1; - } - - if (imm2 < 0) - { - // Set bit location 11 to indicate imm2 is negative - immOut |= 0x800; - imm2 *= -1; - } - - immOut |= imm1; - immOut |= (imm2 << 6); - return immOut; +/*static*/ emitter::code_t emitter::insEncodeReg_V_20_to_16(regNumber reg) +{ + assert(isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 16; } /***************************************************************************** * - * Decodes imm into two 5-bit signed immediates, - * using the encoding format from insEncodeTwoSimm5. + * Return an encoding for the specified 'R' register used in '20' thru '16' position. */ -/*static*/ void emitter::insDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2) -{ - assert(imm1 != nullptr); - assert(imm2 != nullptr); - - *imm1 = (imm & 0x1F); - - if ((imm & 0x20) != 0) - { - *imm1 *= -1; - } - - imm >>= 6; - *imm2 = (imm & 0x1F); - - if ((imm & 0x20) != 0) - { - *imm2 *= -1; - } - assert(isValidSimm<5>(*imm1)); - assert(isValidSimm<5>(*imm2)); +/*static*/ emitter::code_t emitter::insEncodeReg_R_20_to_16(regNumber reg) +{ + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 16; } /***************************************************************************** * - * Returns the encoding to select an insSvePattern + * Return an encoding for the specified 'R' register used in '9' thru '5' position. */ -/*static*/ emitter::code_t emitter::insEncodeSvePattern(insSvePattern pattern) -{ - return (code_t)((unsigned)pattern << 5); -} -BYTE* emitter::emitOutputLoadLabel(BYTE* dst, BYTE* srcAddr, BYTE* dstAddr, instrDescJmp* id) +/*static*/ emitter::code_t emitter::insEncodeReg_R_9_to_5(regNumber reg) { - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - regNumber dstReg = id->idReg1(); - if (id->idjShort) - { - // adr x, [rel addr] -- compute address: current addr(ip) + rel addr. - assert(ins == INS_adr); - assert(fmt == IF_DI_1E); - ssize_t distVal = (ssize_t)(dstAddr - srcAddr); - dst = emitOutputShortAddress(dst, ins, fmt, distVal, dstReg); - } - else - { - // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr - assert(fmt == IF_LARGEADR); - ssize_t relPageAddr = computeRelPageAddr((size_t)dstAddr, (size_t)srcAddr); - dst = emitOutputShortAddress(dst, INS_adrp, IF_DI_1E, relPageAddr, dstReg); - - // add x, x, page offs -- compute address = page addr + page offs - ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits - assert(isValidUimm<12>(imm12)); - code_t code = - emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) - code |= insEncodeDatasize(EA_8BYTE); // X - code |= ((code_t)imm12 << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rd(dstReg); // ddddd - code |= insEncodeReg_Rn(dstReg); // nnnnn - dst += emitOutput_Instr(dst, code); - } - return dst; + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 5; } /***************************************************************************** * - * Output a local jump or other instruction with a pc-relative immediate. - * Note that this may be invoked to overwrite an existing jump instruction at 'dst' - * to handle forward branch patching. + * Return an encoding for the specified 'R' register used in '4' thru '0' position. */ -BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) -{ - instrDescJmp* id = (instrDescJmp*)i; - - unsigned srcOffs; - unsigned dstOffs; - BYTE* srcAddr; - BYTE* dstAddr; - ssize_t distVal; - - // Set default ins/fmt from id. - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - - bool loadLabel = false; - bool isJump = false; - bool loadConstant = false; - - switch (ins) - { - default: - isJump = true; - break; - - case INS_tbz: - case INS_tbnz: - case INS_cbz: - case INS_cbnz: - isJump = true; - break; - - case INS_ldr: - case INS_ldrsw: - loadConstant = true; - break; - - case INS_adr: - case INS_adrp: - loadLabel = true; - break; - } - - /* Figure out the distance to the target */ - - srcOffs = emitCurCodeOffs(dst); - srcAddr = emitOffsetToPtr(srcOffs); - - if (id->idAddr()->iiaIsJitDataOffset()) - { - assert(loadConstant || loadLabel); - int doff = id->idAddr()->iiaGetJitDataOffset(); - assert(doff >= 0); - ssize_t imm = emitGetInsSC(id); - assert((imm >= 0) && (imm < 0x1000)); // 0x1000 is arbitrary, currently 'imm' is always 0 - - unsigned dataOffs = (unsigned)(doff + imm); - assert(dataOffs < emitDataSize()); - dstAddr = emitDataOffsetToPtr(dataOffs); - - regNumber dstReg = id->idReg1(); - regNumber addrReg = dstReg; // an integer register to compute long address. - emitAttr opSize = id->idOpSize(); - - if (loadConstant) - { - if (id->idjShort) - { - // ldr x/v, [rel addr] -- load constant from current addr(ip) + rel addr. - assert(ins == INS_ldr); - assert(fmt == IF_LS_1A); - distVal = (ssize_t)(dstAddr - srcAddr); - dst = emitOutputShortConstant(dst, ins, fmt, distVal, dstReg, opSize); - } - else - { - // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr - assert(fmt == IF_LARGELDC); - ssize_t relPageAddr = computeRelPageAddr((size_t)dstAddr, (size_t)srcAddr); - if (isVectorRegister(dstReg)) - { - // Update addrReg with the reserved integer register - // since we cannot use dstReg (vector) to load constant directly from memory. - - // If loading a 16-byte value, we will need to load directly into dstReg. - // Thus, encode addrReg for the ld1 instruction. - if (opSize == EA_16BYTE) - { - addrReg = encodingSPtoZR(id->idReg2()); - } - else - { - addrReg = id->idReg2(); - } - - assert(isGeneralRegister(addrReg)); - } - - ins = INS_adrp; - fmt = IF_DI_1E; - dst = emitOutputShortAddress(dst, ins, fmt, relPageAddr, addrReg); - - ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits - assert(isValidUimm<12>(imm12)); - - // Special case: emit add + ld1 instructions for loading 16-byte data into vector register. - if (isVectorRegister(dstReg) && (opSize == EA_16BYTE)) - { - const emitAttr elemSize = EA_1BYTE; - const insOpts opt = optMakeArrangement(opSize, elemSize); - - assert(isGeneralRegisterOrSP(addrReg)); - assert(isValidVectorElemsize(elemSize)); - assert(isValidArrangement(opSize, opt)); - - // Calculate page addr + page offs, then emit ld1 instruction. - dst = emitOutputVectorConstant(dst, imm12, dstReg, addrReg, opSize, elemSize); - } - else - { - // ldr x, [x, 0] -- load constant from address into integer register. - ins = INS_ldr; - fmt = IF_LS_2B; - dst = emitOutputShortConstant(dst, ins, fmt, imm12, addrReg, opSize); - - // fmov v, d -- copy constant in integer register to vector register. - // This is needed only for vector constant. - if (addrReg != dstReg) - { - // fmov Vd,Rn DV_2I X00111100X100111 000000nnnnnddddd 1E27 0000 Vd,Rn - // (scalar, from general) - assert(isVectorRegister(dstReg) && isGeneralRegister(addrReg)); - ins = INS_fmov; - fmt = IF_DV_2I; - code_t code = emitInsCode(ins, fmt); - - code |= insEncodeReg_Vd(dstReg); // ddddd - code |= insEncodeReg_Rn(addrReg); // nnnnn - if (id->idOpSize() == EA_8BYTE) - { - code |= 0x80400000; // X ... X - } - dst += emitOutput_Instr(dst, code); - } - } - } - } - else - { - assert(loadLabel); - dst = emitOutputLoadLabel(dst, srcAddr, dstAddr, id); - } - - return dst; - } - - assert(loadLabel || isJump); - - if (id->idAddr()->iiaHasInstrCount()) - { - assert(ig != NULL); - int instrCount = id->idAddr()->iiaGetInstrCount(); - unsigned insNum = emitFindInsNum(ig, id); - if (instrCount < 0) - { - // Backward branches using instruction count must be within the same instruction group. - assert(insNum + 1 >= (unsigned)(-instrCount)); - } - - dstOffs = ig->igOffs + emitFindOffset(ig, (insNum + 1 + instrCount)); - dstAddr = emitOffsetToPtr(dstOffs); - } - else - { - dstOffs = id->idAddr()->iiaIGlabel->igOffs; - dstAddr = emitOffsetToPtr(dstOffs); - } - - distVal = (ssize_t)(dstAddr - srcAddr); +/*static*/ emitter::code_t emitter::insEncodeReg_R_4_to_0(regNumber reg) +{ + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 0; +} - if (dstOffs <= srcOffs) - { -#if DEBUG_EMIT - /* This is a backward jump - distance is known at this point */ +/***************************************************************************** + * + * Return an encoding for the specified 'R' register used in '17' thru '16' position. + */ - if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) - { - size_t blkOffs = id->idjIG->igOffs; +/*static*/ emitter::code_t emitter::insEncodeReg_R_17_to_16(regNumber reg) +{ + assert(isIntegerRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg; + assert((ureg >= 12) && (ureg <= 15)); + return ureg << 16; +} - if (INTERESTING_JUMP_NUM == 0) - printf("[3] Jump %u:\n", id->idDebugOnlyInfo()->idNum); - printf("[3] Jump block is at %08X - %02X = %08X\n", blkOffs, emitOffsAdj, blkOffs - emitOffsAdj); - printf("[3] Jump is at %08X - %02X = %08X\n", srcOffs, emitOffsAdj, srcOffs - emitOffsAdj); - printf("[3] Label block is at %08X - %02X = %08X\n", dstOffs, emitOffsAdj, dstOffs - emitOffsAdj); - } -#endif - } - else - { - /* This is a forward jump - distance will be an upper limit */ +/***************************************************************************** + * + * Return an encoding for the specified 'V' register used in '18' thru '16' position. + */ - emitFwdJumps = true; +/*static*/ emitter::code_t emitter::insEncodeReg_V_18_to_16(regNumber reg) +{ + assert(isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 7)); + return ureg << 16; +} - /* The target offset will be closer by at least 'emitOffsAdj', but only if this - jump doesn't cross the hot-cold boundary. */ +/***************************************************************************** + * + * Return an encoding for the specified 'V' register used in '19' thru '16' position. + */ - if (!emitJumpCrossHotColdBoundary(srcOffs, dstOffs)) - { - dstOffs -= emitOffsAdj; - distVal -= emitOffsAdj; - } +/*static*/ emitter::code_t emitter::insEncodeReg_V_19_to_16(regNumber reg) +{ + assert(isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 16; +} - /* Record the location of the jump for later patching */ +/***************************************************************************** + * + * Return an encoding for the specified 'V' register used in '9' thru '6' position. + */ - id->idjOffs = dstOffs; +/*static*/ emitter::code_t emitter::insEncodeReg_V_9_to_6(regNumber reg) +{ + assert(isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 6; +} - /* Are we overflowing the id->idjOffs bitfield? */ - if (id->idjOffs != dstOffs) - IMPL_LIMITATION("Method is too large"); +/***************************************************************************** + * + * Return an encoding for the specified 'V' register used in '9' thru '6' position with the times two encoding. + * This encoding requires that the register number be divisible by two. + */ -#if DEBUG_EMIT - if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) - { - size_t blkOffs = id->idjIG->igOffs; +/*static*/ emitter::code_t emitter::insEncodeReg_V_9_to_6_Times_Two(regNumber reg) +{ + assert(isVectorRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_V0; + assert(ureg % 2 == 0); + ureg /= 2u; + assert((ureg >= 0) && (ureg <= 31)); + return ureg << 6; +} - if (INTERESTING_JUMP_NUM == 0) - printf("[4] Jump %u:\n", id->idDebugOnlyInfo()->idNum); - printf("[4] Jump block is at %08X\n", blkOffs); - printf("[4] Jump is at %08X\n", srcOffs); - printf("[4] Label block is at %08X - %02X = %08X\n", dstOffs + emitOffsAdj, emitOffsAdj, dstOffs); - } -#endif - } +/***************************************************************************** + * + * Returns an encoding for the specified condition code. + */ -#ifdef DEBUG - if (0 && emitComp->verbose) - { - size_t sz = 4; - int distValSize = id->idjShort ? 4 : 8; - printf("; %s jump [%08X/%03u] from %0*X to %0*X: dist = 0x%08X\n", (dstOffs <= srcOffs) ? "Fwd" : "Bwd", - dspPtr(id), id->idDebugOnlyInfo()->idNum, distValSize, srcOffs + sz, distValSize, dstOffs, distVal); - } -#endif +/*static*/ emitter::code_t emitter::insEncodeCond(insCond cond) +{ + emitter::code_t uimm = (emitter::code_t)cond; + return uimm << 12; +} - /* For forward jumps, record the address of the distance value */ - id->idjTemp.idjAddr = (distVal > 0) ? dst : NULL; +/***************************************************************************** + * + * Returns an encoding for the condition code with the lowest bit inverted (marked by invert() in the + * architecture manual). + */ - assert(insOptsNone(id->idInsOpt())); +/*static*/ emitter::code_t emitter::insEncodeInvertedCond(insCond cond) +{ + emitter::code_t uimm = (emitter::code_t)cond; + uimm ^= 1; // invert the lowest bit + return uimm << 12; +} - if (isJump) - { - if (id->idjShort) - { - // Short conditional/unconditional jump - assert(!id->idjKeepLong); - assert(emitJumpCrossHotColdBoundary(srcOffs, dstOffs) == false); - assert((fmt == IF_BI_0A) || (fmt == IF_BI_0B) || (fmt == IF_BI_1A) || (fmt == IF_BI_1B)); - dst = emitOutputShortBranch(dst, ins, fmt, distVal, id); - } - else - { - // Long conditional/unconditional jump +/***************************************************************************** + * + * Returns an encoding for the specified flags. + */ - if (fmt == IF_LARGEJMP) - { - // This is a pseudo-instruction format representing a large conditional branch, to allow - // us to get a greater branch target range than we can get by using a straightforward conditional - // branch. It is encoded as a short conditional branch that branches around a long unconditional - // branch. - // - // Conceptually, we have: - // - // b L_target - // - // The code we emit is: - // - // b L_not // 4 bytes. Note that we reverse the condition. - // b L_target // 4 bytes - // L_not: - // - // Note that we don't actually insert any blocks: we simply encode "b L_not" as a branch with - // the correct offset. Note also that this works for both integer and floating-point conditions, because - // the condition inversion takes ordered/unordered into account, preserving NaN behavior. For example, - // "GT" (greater than) is inverted to "LE" (less than, equal, or unordered). +/*static*/ emitter::code_t emitter::insEncodeFlags(insCflags flags) +{ + emitter::code_t uimm = (emitter::code_t)flags; + return uimm; +} - instruction reverseIns; - insFormat reverseFmt; +/***************************************************************************** + * + * Returns the encoding for the Shift Count bits to be used for Arm64 encodings + */ - switch (ins) - { - case INS_cbz: - reverseIns = INS_cbnz; - reverseFmt = IF_BI_1A; - break; - case INS_cbnz: - reverseIns = INS_cbz; - reverseFmt = IF_BI_1A; - break; - case INS_tbz: - reverseIns = INS_tbnz; - reverseFmt = IF_BI_1B; - break; - case INS_tbnz: - reverseIns = INS_tbz; - reverseFmt = IF_BI_1B; - break; - default: - reverseIns = emitJumpKindToIns(emitReverseJumpKind(emitInsToJumpKind(ins))); - reverseFmt = IF_BI_0B; - } +/*static*/ emitter::code_t emitter::insEncodeShiftCount(ssize_t imm, emitAttr size) +{ + assert((imm & 0x003F) == imm); + assert(((imm & 0x0020) == 0) || (size == EA_8BYTE)); - dst = emitOutputShortBranch(dst, - reverseIns, // reverse the conditional instruction - reverseFmt, 8, /* 8 bytes from start of this large conditional - pseudo-instruction to L_not. */ - id); + return (emitter::code_t)imm << 10; +} - // Now, pretend we've got a normal unconditional branch, and fall through to the code to emit that. - ins = INS_b; - fmt = IF_BI_0A; +/***************************************************************************** + * + * Returns the encoding to select a 64-bit datasize for an Arm64 instruction + */ - // The distVal was computed based on the beginning of the pseudo-instruction, - // So subtract the size of the conditional branch so that it is relative to the - // unconditional branch. - distVal -= 4; - } +/*static*/ emitter::code_t emitter::insEncodeDatasize(emitAttr size) +{ + if (size == EA_8BYTE) + { + return 0x80000000; // set the bit at location 31 + } + else + { + assert(size == EA_4BYTE); + return 0; + } +} - assert(fmt == IF_BI_0A); - assert((distVal & 1) == 0); - code_t code = emitInsCode(ins, fmt); - const bool doRecordRelocation = emitComp->opts.compReloc && emitJumpCrossHotColdBoundary(srcOffs, dstOffs); +/***************************************************************************** + * + * Returns the encoding to select the datasize for the general load/store Arm64 instructions + * + */ - if (doRecordRelocation) +/*static*/ emitter::code_t emitter::insEncodeDatasizeLS(emitter::code_t code, emitAttr size) +{ + bool exclusive = ((code & 0x35000000) == 0); + bool atomic = ((code & 0x31200C00) == 0x30200000); + + if ((code & 0x00800000) && !exclusive && !atomic) // Is this a sign-extending opcode? (i.e. ldrsw, ldrsh, ldrsb) + { + if ((code & 0x80000000) == 0) // Is it a ldrsh or ldrsb and not ldrsw ? + { + if (EA_SIZE(size) != EA_8BYTE) // Do we need to encode the 32-bit Rt size bit? { - // dst isn't an actual final target location, just some intermediate - // location. Thus we cannot make any guarantees about distVal (not - // even the direction/sign). Instead we don't encode any offset and - // rely on the relocation to do all the work + return 0x00400000; // set the bit at location 22 } - else - { - // Branch offset encodings are scaled by 4. - noway_assert((distVal & 3) == 0); - distVal >>= 2; - noway_assert(isValidSimm<26>(distVal)); + } + } + else if (code & 0x80000000) // Is this a ldr/str/ldur/stur opcode? + { + if (EA_SIZE(size) == EA_8BYTE) // Do we need to encode the 64-bit size bit? + { + return 0x40000000; // set the bit at location 30 + } + } + return 0; +} - // Insert offset into unconditional branch instruction - distVal &= 0x3FFFFFFLL; - code |= distVal; - } +/***************************************************************************** + * + * Returns the encoding to select the datasize for the vector load/store Arm64 instructions + * + */ - const unsigned instrSize = emitOutput_Instr(dst, code); +/*static*/ emitter::code_t emitter::insEncodeDatasizeVLS(emitter::code_t code, emitAttr size) +{ + code_t result = 0; - if (doRecordRelocation) - { - assert(id->idjKeepLong); - if (emitComp->info.compMatchedVM) - { - void* target = emitOffsetToPtr(dstOffs); - emitRecordRelocation((void*)dst, target, IMAGE_REL_ARM64_BRANCH26); - } - } + // Check bit 29 + if ((code & 0x20000000) == 0) + { + // LDR literal - dst += instrSize; + if (size == EA_16BYTE) + { + // set the operation size in bit 31 + result = 0x80000000; + } + else if (size == EA_8BYTE) + { + // set the operation size in bit 30 + result = 0x40000000; + } + else + { + assert(size == EA_4BYTE); + // no bits are set + result = 0x00000000; } } - else if (loadLabel) + else { - dst = emitOutputLoadLabel(dst, srcAddr, dstAddr, id); + // LDR non-literal + + if (size == EA_16BYTE) + { + // The operation size in bits 31 and 30 are zero + // Bit 23 specifies a 128-bit Load/Store + result = 0x00800000; + } + else if (size == EA_8BYTE) + { + // set the operation size in bits 31 and 30 + result = 0xC0000000; + } + else if (size == EA_4BYTE) + { + // set the operation size in bit 31 + result = 0x80000000; + } + else if (size == EA_2BYTE) + { + // set the operation size in bit 30 + result = 0x40000000; + } + else + { + assert(size == EA_1BYTE); + // The operation size in bits 31 and 30 are zero + result = 0x00000000; + } } - return dst; + // Or in bit 26 to indicate a Vector register is used as 'target' + result |= 0x04000000; + + return result; } /***************************************************************************** -* -* Output a short branch instruction. -*/ -BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, instrDescJmp* id) -{ - code_t code = emitInsCode(ins, fmt); + * + * Returns the encoding to select the datasize for the vector load/store Arm64 instructions + * + */ - ssize_t loBits = (distVal & 3); - noway_assert(loBits == 0); - distVal >>= 2; // branch offset encodings are scaled by 4. +/*static*/ emitter::code_t emitter::insEncodeDatasizeVPLS(emitter::code_t code, emitAttr size) +{ + code_t result = 0; - if (fmt == IF_BI_0A) + if (size == EA_16BYTE) { - // INS_b or INS_bl_local - noway_assert(isValidSimm<26>(distVal)); - distVal &= 0x3FFFFFFLL; - code |= distVal; + // The operation size in bits 31 and 30 are zero + // Bit 23 specifies a 128-bit Load/Store + result = 0x80000000; } - else if (fmt == IF_BI_0B) // BI_0B 01010100iiiiiiii iiiiiiiiiiiXXXXX simm19:00 + else if (size == EA_8BYTE) { - // INS_beq, INS_bne, etc... - noway_assert(isValidSimm<19>(distVal)); - distVal &= 0x7FFFFLL; - code |= distVal << 5; + // set the operation size in bits 31 and 30 + result = 0x40000000; } - else if (fmt == IF_BI_1A) // BI_1A X.......iiiiiiii iiiiiiiiiiittttt Rt simm19:00 + else if (size == EA_4BYTE) { - // INS_cbz or INS_cbnz - assert(id != nullptr); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - - noway_assert(isValidSimm<19>(distVal)); - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; + // set the operation size in bit 31 + result = 0x00000000; } - else if (fmt == IF_BI_1B) // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 - { - // INS_tbz or INS_tbnz - assert(id != nullptr); - ssize_t imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - if (imm & 0x20) // test bit 32-63 ? + // Or in bit 26 to indicate a Vector register is used as 'target' + result |= 0x04000000; + + return result; +} + +/***************************************************************************** + * + * Returns the encoding to set the size bit and the N bits for a 'bitfield' instruction + * + */ + +/*static*/ emitter::code_t emitter::insEncodeDatasizeBF(emitter::code_t code, emitAttr size) +{ + // is bit 30 equal to 0? + if ((code & 0x40000000) == 0) // is the opcode one of extr, sxtb, sxth or sxtw + { + if (size == EA_8BYTE) // Do we need to set the sf and N bits? { - code |= 0x80000000; // B + return 0x80400000; // set the sf-bit at location 31 and the N-bit at location 22 } - code |= ((imm & 0x1F) << 19); // bbbbb - code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + return 0; // don't set any bits +} - noway_assert(isValidSimm<14>(distVal)); - distVal &= 0x3FFFLL; // 14 bits - code |= distVal << 5; +/***************************************************************************** + * + * Returns the encoding to select the 64/128-bit datasize for an Arm64 vector instruction + */ + +/*static*/ emitter::code_t emitter::insEncodeVectorsize(emitAttr size) +{ + if (size == EA_16BYTE) + { + return 0x40000000; // set the bit at location 30 } else { - assert(!"Unknown fmt for emitOutputShortBranch"); + assert(size == EA_8BYTE); + return 0; } +} - dst += emitOutput_Instr(dst, code); +/***************************************************************************** + * + * Returns the encoding to select 'index' for an Arm64 vector elem instruction + */ +/*static*/ emitter::code_t emitter::insEncodeVectorIndex(emitAttr elemsize, ssize_t index) +{ + code_t bits = (code_t)index; + if (elemsize == EA_1BYTE) + { + bits <<= 1; + bits |= 1; + } + else if (elemsize == EA_2BYTE) + { + bits <<= 2; + bits |= 2; + } + else if (elemsize == EA_4BYTE) + { + bits <<= 3; + bits |= 4; + } + else + { + assert(elemsize == EA_8BYTE); + bits <<= 4; + bits |= 8; + } + assert((bits >= 1) && (bits <= 0x1f)); - return dst; + return (bits << 16); // bits at locations [20,19,18,17,16] } /***************************************************************************** -* -* Output a short address instruction. -*/ -BYTE* emitter::emitOutputShortAddress(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, regNumber reg) + * + * Returns the encoding to select 'index2' for an Arm64 'ins' elem instruction + */ +/*static*/ emitter::code_t emitter::insEncodeVectorIndex2(emitAttr elemsize, ssize_t index2) { - ssize_t loBits = (distVal & 3); - distVal >>= 2; - - code_t code = emitInsCode(ins, fmt); - if (fmt == IF_DI_1E) // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 + code_t bits = (code_t)index2; + if (elemsize == EA_1BYTE) + { + // bits are correct + } + else if (elemsize == EA_2BYTE) { - // INS_adr or INS_adrp - code |= insEncodeReg_Rd(reg); // ddddd - - noway_assert(isValidSimm<19>(distVal)); - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; - code |= loBits << 29; // 2 bits + bits <<= 1; + } + else if (elemsize == EA_4BYTE) + { + bits <<= 2; } else { - assert(!"Unknown fmt for emitOutputShortAddress"); + assert(elemsize == EA_8BYTE); + bits <<= 3; } + assert((bits >= 0) && (bits <= 0xf)); - dst += emitOutput_Instr(dst, code); - - return dst; + return (bits << 11); // bits at locations [14,13,12,11] } /***************************************************************************** -* -* Output a short constant instruction. -*/ -BYTE* emitter::emitOutputShortConstant( - BYTE* dst, instruction ins, insFormat fmt, ssize_t imm, regNumber reg, emitAttr opSize) + * + * Returns the encoding to select the 'index' for an Arm64 'mul' by element instruction + */ +/*static*/ emitter::code_t emitter::insEncodeVectorIndexLMH(emitAttr elemsize, ssize_t index) { - code_t code = emitInsCode(ins, fmt); + code_t bits = 0; - if (fmt == IF_LS_1A) + if (elemsize == EA_2BYTE) { - // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt simm21 - // INS_ldr or INS_ldrsw (PC-Relative) - - ssize_t loBits = (imm & 3); - noway_assert(loBits == 0); - ssize_t distVal = imm >> 2; // load offset encodings are scaled by 4. - - noway_assert(isValidSimm<19>(distVal)); - - // Is the target a vector register? - if (isVectorRegister(reg)) + assert((index >= 0) && (index <= 7)); + if (index & 0x4) { - code |= insEncodeDatasizeVLS(code, opSize); // XX V - code |= insEncodeReg_Vt(reg); // ttttt + bits |= (1 << 11); // set bit 11 'H' } - else + if (index & 0x2) { - assert(isGeneralRegister(reg)); - // insEncodeDatasizeLS is not quite right for this case. - // So just specialize it. - if ((ins == INS_ldr) && (opSize == EA_8BYTE)) - { - // set the operation size in bit 30 - code |= 0x40000000; - } - - code |= insEncodeReg_Rt(reg); // ttttt + bits |= (1 << 21); // set bit 21 'L' + } + if (index & 0x1) + { + bits |= (1 << 20); // set bit 20 'M' } - - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; } - else if (fmt == IF_LS_2B) + else if (elemsize == EA_4BYTE) { - // ldr Rt,[Xn+pimm12] LS_2B 1X11100101iiiiii iiiiiinnnnnttttt B940 0000 imm(0-4095<<{2,3}) - // INS_ldr or INS_ldrsw (PC-Relative) - noway_assert(isValidUimm<12>(imm)); - assert(isGeneralRegister(reg)); - - if (opSize == EA_8BYTE) + assert((index >= 0) && (index <= 3)); + if (index & 0x2) { - // insEncodeDatasizeLS is not quite right for this case. - // So just specialize it. - if (ins == INS_ldr) - { - // set the operation size in bit 30 - code |= 0x40000000; - } - // Low 3 bits should be 0 -- 8 byte JIT data should be aligned on 8 byte. - assert((imm & 7) == 0); - imm >>= 3; + bits |= (1 << 11); // set bit 11 'H' } - else + if (index & 0x1) { - assert(opSize == EA_4BYTE); - // Low 2 bits should be 0 -- 4 byte aligned data. - assert((imm & 3) == 0); - imm >>= 2; + bits |= (1 << 21); // set bit 21 'L' } - - code |= insEncodeReg_Rt(reg); // ttttt - code |= insEncodeReg_Rn(reg); // nnnnn - code |= imm << 10; } else { - assert(!"Unknown fmt for emitOutputShortConstant"); + assert(!"Invalid 'elemsize' value"); } - dst += emitOutput_Instr(dst, code); - - return dst; + return bits; } /***************************************************************************** * - * Output instructions to load a constant into a vector register. + * Returns the encoding for a shift instruction, ready for insertion into an instruction. */ -BYTE* emitter::emitOutputVectorConstant( - BYTE* dst, ssize_t imm, regNumber dstReg, regNumber addrReg, emitAttr opSize, emitAttr elemSize) +/*static*/ emitter::code_t emitter::insEncodeShiftImmediate(emitAttr size, bool isRightShift, ssize_t shiftAmount) { - // add addrReg, addrReg, page offs -- compute address = page addr + page offs. - code_t code = emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) - code |= insEncodeDatasize(EA_8BYTE); // X - use EA_8BYTE, as we are calculating 64-bit address - code |= ((code_t)imm << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rd(addrReg); // ddddd - code |= insEncodeReg_Rn(addrReg); // nnnnn - dst += emitOutput_Instr(dst, code); - - // ld1 dstReg, addrReg -- load constant at address in addrReg into dstReg. - code = emitInsCode(INS_ld1, IF_LS_2D); // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn - code |= insEncodeVectorsize(opSize); // Q - code |= insEncodeVLSElemsize(elemSize); // ss - code |= insEncodeReg_Rn(addrReg); // nnnnn - code |= insEncodeReg_Vt(dstReg); // ttttt - dst += emitOutput_Instr(dst, code); - - return dst; + if (isRightShift) + { + // The right shift amount must be in the range 1 to the destination element width in bits. + assert((shiftAmount > 0) && (shiftAmount <= getBitWidth(size))); + return (code_t)(2 * getBitWidth(size) - shiftAmount); + } + else + { + // The left shift amount must in the range 0 to the element width in bits minus 1. + assert(shiftAmount < getBitWidth(size)); + return (code_t)(getBitWidth(size) + shiftAmount); + } } /***************************************************************************** * - * Output a call instruction. + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 vector instruction */ -unsigned emitter::emitOutputCall(insGroup* ig, BYTE* dst, instrDesc* id, code_t code) +/*static*/ emitter::code_t emitter::insEncodeElemsize(emitAttr size) { - const unsigned char callInstrSize = sizeof(code_t); // 4 bytes - regMaskTP gcrefRegs; - regMaskTP byrefRegs; - - VARSET_TP GCvars(VarSetOps::UninitVal()); - - // Is this a "fat" call descriptor? - if (id->idIsLargeCall()) + if (size == EA_8BYTE) { - instrDescCGCA* idCall = (instrDescCGCA*)id; - gcrefRegs = idCall->idcGcrefRegs; - byrefRegs = idCall->idcByrefRegs; - VarSetOps::Assign(emitComp, GCvars, idCall->idcGCvars); + return 0x00C00000; // set the bit at location 23 and 22 } - else + else if (size == EA_4BYTE) { - assert(!id->idIsLargeDsp()); - assert(!id->idIsLargeCns()); - - gcrefRegs = emitDecodeCallGCregs(id); - byrefRegs = 0; - VarSetOps::AssignNoCopy(emitComp, GCvars, VarSetOps::MakeEmpty(emitComp)); + return 0x00800000; // set the bit at location 23 } - - /* We update the GC info before the call as the variables cannot be - used by the call. Killing variables before the call helps with - boundary conditions if the call is CORINFO_HELP_THROW - see bug 50029. - If we ever track aliased variables (which could be used by the - call), we would have to keep them alive past the call. */ - - emitUpdateLiveGCvars(GCvars, dst); - -#ifdef DEBUG - // Output any delta in GC variable info, corresponding to the before-call GC var updates done above. - if (EMIT_GC_VERBOSE || emitComp->opts.disasmWithGC) + else if (size == EA_2BYTE) { - emitDispGCVarDelta(); + return 0x00400000; // set the bit at location 22 } -#endif // DEBUG - - // Now output the call instruction and update the 'dst' pointer - // - unsigned outputInstrSize = emitOutput_Instr(dst, code); - dst += outputInstrSize; + assert(size == EA_1BYTE); + return 0x00000000; +} - // All call instructions are 4-byte in size on ARM64 - // - assert(outputInstrSize == callInstrSize); +/***************************************************************************** + * + * Returns the encoding to select the 4/8 byte elemsize for an Arm64 float vector instruction + */ - // If the method returns a GC ref, mark INTRET (R0) appropriately. - if (id->idGCref() == GCT_GCREF) - { - gcrefRegs |= RBM_INTRET; - } - else if (id->idGCref() == GCT_BYREF) +/*static*/ emitter::code_t emitter::insEncodeFloatElemsize(emitAttr size) +{ + if (size == EA_8BYTE) { - byrefRegs |= RBM_INTRET; + return 0x00400000; // set the bit at location 22 } + assert(size == EA_4BYTE); + return 0x00000000; +} - // If is a multi-register return method is called, mark INTRET_1 (X1) appropriately - if (id->idIsLargeCall()) +// Returns the encoding to select the index for an Arm64 float vector by element instruction +/*static*/ emitter::code_t emitter::insEncodeFloatIndex(emitAttr elemsize, ssize_t index) +{ + code_t result = 0x00000000; + if (elemsize == EA_8BYTE) { - instrDescCGCA* idCall = (instrDescCGCA*)id; - if (idCall->idSecondGCref() == GCT_GCREF) - { - gcrefRegs |= RBM_INTRET_1; - } - else if (idCall->idSecondGCref() == GCT_BYREF) + assert((index >= 0) && (index <= 1)); + if (index == 1) { - byrefRegs |= RBM_INTRET_1; + result |= 0x00000800; // 'H' - set the bit at location 11 } } - - // If the GC register set has changed, report the new set. - if (gcrefRegs != emitThisGCrefRegs) - { - emitUpdateLiveGCregs(GCT_GCREF, gcrefRegs, dst); - } - // If the Byref register set has changed, report the new set. - if (byrefRegs != emitThisByrefRegs) - { - emitUpdateLiveGCregs(GCT_BYREF, byrefRegs, dst); - } - - // Some helper calls may be marked as not requiring GC info to be recorded. - if ((!id->idIsNoGC())) - { - // On ARM64, as on AMD64, we don't change the stack pointer to push/pop args. - // So we're not really doing a "stack pop" here (note that "args" is 0), but we use this mechanism - // to record the call for GC info purposes. (It might be best to use an alternate call, - // and protect "emitStackPop" under the EMIT_TRACK_STACK_DEPTH preprocessor variable.) - emitStackPop(dst, /*isCall*/ true, callInstrSize, /*args*/ 0); - - // Do we need to record a call location for GC purposes? - // - if (!emitFullGCinfo) + else + { + assert(elemsize == EA_4BYTE); + assert((index >= 0) && (index <= 3)); + if (index & 2) { - emitRecordGCcall(dst, callInstrSize); + result |= 0x00000800; // 'H' - set the bit at location 11 + } + if (index & 1) + { + result |= 0x00200000; // 'L' - set the bit at location 21 } } - return callInstrSize; + return result; } /***************************************************************************** * - * Emit a 32-bit Arm64 instruction - */ - -unsigned emitter::emitOutput_Instr(BYTE* dst, code_t code) -{ - assert(sizeof(code_t) == 4); - BYTE* dstRW = dst + writeableOffset; - *((code_t*)dstRW) = code; - - return sizeof(code_t); -} - -/***************************************************************************** -* - * Append the machine code corresponding to the given instruction descriptor - * to the code block at '*dp'; the base of the code block is 'bp', and 'ig' - * is the instruction group that contains the instruction. Updates '*dp' to - * point past the generated code, and returns the size of the instruction - * descriptor in bytes. + * Returns the encoding to select the vector elemsize for an Arm64 ld/st# vector instruction */ -size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) +/*static*/ emitter::code_t emitter::insEncodeVLSElemsize(emitAttr size) { - BYTE* dst = *dp; - BYTE* odst = dst; - code_t code = 0; - size_t sz = emitGetInstrDescSize(id); // TODO-ARM64-Cleanup: on ARM, this is set in each case. why? - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - emitAttr size = id->idOpSize(); - -#ifdef DEBUG -#if DUMP_GC_TABLES - bool dspOffs = emitComp->opts.dspGCtbls; -#else - bool dspOffs = !emitComp->opts.disDiffable; -#endif -#endif // DEBUG - - assert(REG_NA == (int)REG_NA); - - /* What instruction format have we got? */ + code_t result = 0x00000000; - switch (fmt) + switch (size) { - ssize_t imm; - ssize_t index; - ssize_t index2; - unsigned cmode; - unsigned immShift; - emitAttr elemsize; - emitAttr datasize; - - case IF_BI_0A: // BI_0A ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 - case IF_BI_0B: // BI_0B ......iiiiiiiiii iiiiiiiiiii..... simm19:00 - case IF_LARGEJMP: - assert(id->idGCref() == GCT_NONE); - assert(id->idIsBound()); - dst = emitOutputLJ(ig, dst, id); - sz = sizeof(instrDescJmp); + case EA_1BYTE: + { + result |= 0x0000; // clear bits 10 and 11 break; + } - case IF_BI_0C: // BI_0C ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 - code = emitInsCode(ins, fmt); - sz = id->idIsLargeCall() ? sizeof(instrDescCGCA) : sizeof(instrDesc); - dst += emitOutputCall(ig, dst, id, code); - // Always call RecordRelocation so that we wire in a JumpStub when we don't reach - emitRecordRelocation(odst, id->idAddr()->iiaAddr, IMAGE_REL_ARM64_BRANCH26); + case EA_2BYTE: + { + result |= 0x0400; // set bit at location 10, clear bit at location 11 break; + } - case IF_BI_1A: // BI_1A ......iiiiiiiiii iiiiiiiiiiittttt Rt simm19:00 - assert(insOptsNone(id->idInsOpt())); - assert(id->idIsBound()); - - dst = emitOutputLJ(ig, dst, id); - sz = sizeof(instrDescJmp); + case EA_4BYTE: + { + result |= 0x0800; // clear bit at location 10, set bit at location 11 break; + } - case IF_BI_1B: // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 - assert(insOptsNone(id->idInsOpt())); - assert(id->idIsBound()); - - dst = emitOutputLJ(ig, dst, id); - sz = sizeof(instrDescJmp); + case EA_8BYTE: + { + result |= 0x0C00; // set bits at location 10 and 11 break; + } - case IF_BR_1A: // BR_1A ................ ......nnnnn..... Rn - assert(insOptsNone(id->idInsOpt())); - assert((ins == INS_ret) || (ins == INS_br)); - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - - dst += emitOutput_Instr(dst, code); + default: + { + assert(!"Invalid element size"); break; + } + } - case IF_BR_1B: // BR_1B ................ ......nnnnn..... Rn - assert(insOptsNone(id->idInsOpt())); - assert((ins == INS_br_tail) || (ins == INS_blr)); - code = emitInsCode(ins, fmt); + return result; +} - if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && id->idIsTlsGD()) - { - emitRecordRelocation(odst, (CORINFO_METHOD_HANDLE)id->idAddr()->iiaAddr, - IMAGE_REL_AARCH64_TLSDESC_CALL); - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - } - else - { - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - } - dst += emitOutputCall(ig, dst, id, code); - sz = id->idIsLargeCall() ? sizeof(instrDescCGCA) : sizeof(instrDesc); - break; +/***************************************************************************** + * + * Returns the encoding to select the index for an Arm64 ld/st# vector by element instruction + */ - case IF_LS_1A: // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt PC imm(1MB) - case IF_LARGELDC: - assert(insOptsNone(id->idInsOpt())); - assert(id->idIsBound()); +/*static*/ emitter::code_t emitter::insEncodeVLSIndex(emitAttr size, ssize_t index) +{ + code_t result = 0x00000000; - dst = emitOutputLJ(ig, dst, id); - sz = sizeof(instrDescJmp); - break; + switch (size) + { + case EA_1BYTE: + { + // Q = ? - bit location 30 + // xx = 00 - bit location 14 and 15 + // S = ? - bit location 12 + // ss = ?0 - bit location 10 and 11 - case IF_LS_2A: // LS_2A .X.......X...... ......nnnnnttttt Rt Rn - assert(insOptsNone(id->idInsOpt())); - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - if (id->idIsTlsGD()) - { - emitRecordRelocation(odst, (void*)emitGetInsSC(id), IMAGE_REL_AARCH64_TLSDESC_LD64_LO12); - } + result |= (index & 0x8) << 27; + result |= (index & 0x4) << 10; + result |= (index & 0x3) << 10; break; + } - case IF_LS_2B: // LS_2B .X.......Xiiiiii iiiiiinnnnnttttt Rt Rn imm(0-4095) - assert(insOptsNone(id->idInsOpt())); - imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } - code |= ((code_t)imm << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; + case EA_2BYTE: + { + // Q = ? - bit location 30 + // xx = 01 - bit location 14 and 15 + // S = ? - bit location 12 + // ss = ?? - bit location 10 and 11 - case IF_LS_2C: // LS_2C .X.......X.iiiii iiiiPPnnnnnttttt Rt Rn imm(-256..+255) no/pre/post inc - assert(insOptsNone(id->idInsOpt()) || insOptsIndexed(id->idInsOpt())); - imm = emitGetInsSC(id); - assert((imm >= -256) && (imm <= 255)); // signed 9 bits - imm &= 0x1ff; // force into unsigned 9 bit representation - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } - code |= insEncodeIndexedOpt(id->idInsOpt()); // PP - code |= ((code_t)imm << 12); // iiiiiiiii - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); + result |= (index & 0x4) << 28; + result |= 0x4000; + result |= (index & 0x2) << 11; + result |= (index & 0x1) << 11; break; + } - case IF_LS_2D: // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn - case IF_LS_2E: // LS_2E .Q.............. ....ssnnnnnttttt Vt Rn - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeVLSElemsize(elemsize); // ss - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vt(id->idReg1()); // ttttt + case EA_4BYTE: + { + // Q = ? - bit location 30 + // xx = 10 - bit location 14 and 15 + // S = ? - bit location 12 + // ss = 00 - bit location 10 and 11 - dst += emitOutput_Instr(dst, code); + result |= (index & 0x2) << 29; + result |= 0x8000; + result |= (index & 0x1) << 12; break; + } - case IF_LS_2F: // LS_2F .Q.............. xx.Sssnnnnnttttt Vt[] Rn - case IF_LS_2G: // LS_2G .Q.............. xx.Sssnnnnnttttt Vt[] Rn - elemsize = id->idOpSize(); - index = id->idSmallCns(); - code = emitInsCode(ins, fmt); - - code |= insEncodeVLSIndex(elemsize, index); // Q xx S ss - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vt(id->idReg1()); // ttttt + case EA_8BYTE: + { + // Q = ? - bit location 30 + // xx = 10 - bit location 14 and 15 + // S = 0 - bit location 12 + // ss = 01 - bit location 10 and 11 - dst += emitOutput_Instr(dst, code); + result |= (index & 0x1) << 30; + result |= 0x8400; break; + } - case IF_LS_3A: // LS_3A .X.......X.mmmmm oooS..nnnnnttttt Rt Rn Rm ext(Rm) LSL {} - assert(insOptsLSExtend(id->idInsOpt())); - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } - code |= insEncodeExtend(id->idInsOpt()); // ooo - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - if (id->idIsLclVar()) - { - code |= insEncodeReg_Rm(codeGen->rsGetRsvdReg()); // mmmmm - } - else - { - code |= insEncodeReg3Scale(id->idReg3Scaled()); // S - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - } - dst += emitOutput_Instr(dst, code); + default: + { + assert(!"Invalid element size"); break; + } + } - case IF_LS_3B: // LS_3B X............... .aaaaannnnnddddd Rd Ra Rn - assert(insOptsNone(id->idInsOpt())); - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVPLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - code |= insEncodeReg_Va(id->idReg2()); // aaaaa - } - else - { - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - code |= insEncodeReg_Ra(id->idReg2()); // aaaaa - } - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); + return result; +} + +/***************************************************************************** + * + * Returns the encoding to select the fcvt operation for Arm64 instructions + */ +/*static*/ emitter::code_t emitter::insEncodeConvertOpt(insFormat fmt, insOpts conversion) +{ + code_t result = 0; + switch (conversion) + { + case INS_OPTS_S_TO_D: // Single to Double + assert(fmt == IF_DV_2J); + result = 0x00008000; // type=00, opc=01 break; - case IF_LS_3C: // LS_3C X......PP.iiiiii iaaaaannnnnddddd Rd Ra Rn imm(im7,sh) - assert(insOptsNone(id->idInsOpt()) || insOptsIndexed(id->idInsOpt())); - imm = emitGetInsSC(id); - assert((imm >= -64) && (imm <= 63)); // signed 7 bits - imm &= 0x7f; // force into unsigned 7 bit representation - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVPLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - code |= insEncodeReg_Va(id->idReg2()); // aaaaa - } - else - { - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - code |= insEncodeReg_Ra(id->idReg2()); // aaaaa - } - code |= insEncodePairIndexedOpt(ins, id->idInsOpt()); // PP - code |= ((code_t)imm << 15); // iiiiiiiii - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); + case INS_OPTS_D_TO_S: // Double to Single + assert(fmt == IF_DV_2J); + result = 0x00400000; // type=01, opc=00 break; - case IF_LS_3D: // LS_3D .X.......X.mmmmm ......nnnnnttttt Wm Rt Rn - code = emitInsCode(ins, fmt); - // Arm64 store exclusive unpredictable cases - assert(id->idReg1() != id->idReg2()); - assert(id->idReg1() != id->idReg3()); - code |= insEncodeDatasizeLS(code, id->idOpSize()); // X - code |= insEncodeReg_Rm(id->idReg1()); // mmmmm - code |= insEncodeReg_Rt(id->idReg2()); // ttttt - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); + case INS_OPTS_H_TO_S: // Half to Single + assert(fmt == IF_DV_2J); + result = 0x00C00000; // type=11, opc=00 break; - case IF_LS_3E: // LS_3E .X.........mmmmm ......nnnnnttttt Rm Rt Rn ARMv8.1 LSE Atomics - code = emitInsCode(ins, fmt); - code |= insEncodeDatasizeLS(code, id->idOpSize()); // X - code |= insEncodeReg_Rm(id->idReg1()); // mmmmm - code |= insEncodeReg_Rt(id->idReg2()); // ttttt - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); + case INS_OPTS_H_TO_D: // Half to Double + assert(fmt == IF_DV_2J); + result = 0x00C08000; // type=11, opc=01 + break; - // Some instructions with this encoding return their result in the - // second operand register instead of the first so we special case - // the GC update here and skip the common path down below. - if (emitInsDestIsOp2(ins)) - { - if (id->idGCref() != GCT_NONE) - { - emitGCregLiveUpd(id->idGCref(), id->idReg2(), dst); - } - else - { - emitGCregDeadUpd(id->idReg2(), dst); - } + case INS_OPTS_S_TO_H: // Single to Half + assert(fmt == IF_DV_2J); + result = 0x00018000; // type=00, opc=11 + break; - goto SKIP_GC_UPDATE; - } + case INS_OPTS_D_TO_H: // Double to Half + assert(fmt == IF_DV_2J); + result = 0x00418000; // type=01, opc=11 + break; + case INS_OPTS_S_TO_4BYTE: // Single to INT32 + assert(fmt == IF_DV_2H); + result = 0x00000000; // sf=0, type=00 break; - case IF_LS_3F: // LS_3F .Q.........mmmmm ....ssnnnnnttttt Vt Rn Rm - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); + case INS_OPTS_D_TO_4BYTE: // Double to INT32 + assert(fmt == IF_DV_2H); + result = 0x00400000; // sf=0, type=01 + break; - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeVLSElemsize(elemsize); // ss - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vt(id->idReg1()); // ttttt + case INS_OPTS_S_TO_8BYTE: // Single to INT64 + assert(fmt == IF_DV_2H); + result = 0x80000000; // sf=1, type=00 + break; - dst += emitOutput_Instr(dst, code); + case INS_OPTS_D_TO_8BYTE: // Double to INT64 + assert(fmt == IF_DV_2H); + result = 0x80400000; // sf=1, type=01 break; - case IF_LS_3G: // LS_3G .Q.........mmmmm ...Sssnnnnnttttt Vt[] Rn Rm - elemsize = id->idOpSize(); - index = id->idSmallCns(); - code = emitInsCode(ins, fmt); + case INS_OPTS_4BYTE_TO_S: // INT32 to Single + assert(fmt == IF_DV_2I); + result = 0x00000000; // sf=0, type=00 + break; - code |= insEncodeVLSIndex(elemsize, index); // Q xx S ss - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vt(id->idReg1()); // ttttt + case INS_OPTS_4BYTE_TO_D: // INT32 to Double + assert(fmt == IF_DV_2I); + result = 0x00400000; // sf=0, type=01 + break; - dst += emitOutput_Instr(dst, code); + case INS_OPTS_8BYTE_TO_S: // INT64 to Single + assert(fmt == IF_DV_2I); + result = 0x80000000; // sf=1, type=00 break; - case IF_DI_1A: // DI_1A X.......shiiiiii iiiiiinnnnn..... Rn imm(i12,sh) - assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); - imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeShiftImm12(id->idInsOpt()); // sh - code |= ((code_t)imm << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - dst += emitOutput_Instr(dst, code); + case INS_OPTS_8BYTE_TO_D: // INT64 to Double + assert(fmt == IF_DV_2I); + result = 0x80400000; // sf=1, type=01 break; - case IF_DI_1B: // DI_1B X........hwiiiii iiiiiiiiiiiddddd Rd imm(i16,hw) - imm = emitGetInsSC(id); - assert(isValidImmHWVal(imm, id->idOpSize())); - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= ((code_t)imm << 5); // hwiiiii iiiiiiiiiii - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); + default: + assert(!"Invalid 'conversion' value"); break; + } + return result; +} + +/***************************************************************************** + * + * Returns the encoding to have the Rn register be updated Pre/Post indexed + * or not updated + */ + +/*static*/ emitter::code_t emitter::insEncodeIndexedOpt(insOpts opt) +{ + assert(emitter::insOptsNone(opt) || emitter::insOptsIndexed(opt)); + + if (emitter::insOptsIndexed(opt)) + { + if (emitter::insOptsPostIndex(opt)) + { + return 0x00000400; // set the bit at location 10 + } + else + { + assert(emitter::insOptsPreIndex(opt)); + return 0x00000C00; // set the bit at location 10 and 11 + } + } + else + { + assert(emitter::insOptsNone(opt)); + return 0; // bits 10 and 11 are zero + } +} - case IF_DI_1C: // DI_1C X........Nrrrrrr ssssssnnnnn..... Rn imm(N,r,s) - imm = emitGetInsSC(id); - assert(isValidImmNRS(imm, id->idOpSize())); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 10); // Nrrrrrrssssss - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for a ldp/stp instruction to have the Rn register + * be updated Pre/Post indexed or not updated + */ - case IF_DI_1D: // DI_1D X........Nrrrrrr ssssss.....ddddd Rd imm(N,r,s) - imm = emitGetInsSC(id); - assert(isValidImmNRS(imm, id->idOpSize())); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 10); // Nrrrrrrssssss - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodePairIndexedOpt(instruction ins, insOpts opt) +{ + assert(emitter::insOptsNone(opt) || emitter::insOptsIndexed(opt)); - case IF_DI_1E: // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 - case IF_LARGEADR: - assert(insOptsNone(id->idInsOpt())); - if (id->idIsReloc()) + if ((ins == INS_ldnp) || (ins == INS_stnp)) + { + assert(emitter::insOptsNone(opt)); + return 0; // bits 23 and 24 are zero + } + else + { + if (emitter::insOptsIndexed(opt)) + { + if (emitter::insOptsPostIndex(opt)) { - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21 - : IMAGE_REL_ARM64_PAGEBASE_REL21); + return 0x00800000; // set the bit at location 23 } else { - // Local jmp/load case which does not need a relocation. - assert(id->idIsBound()); - dst = emitOutputLJ(ig, dst, id); + assert(emitter::insOptsPreIndex(opt)); + return 0x01800000; // set the bit at location 24 and 23 } - sz = sizeof(instrDescJmp); - break; + } + else + { + assert(emitter::insOptsNone(opt)); + return 0x01000000; // set the bit at location 24 + } + } +} - case IF_DI_1F: // DI_1F X..........iiiii cccc..nnnnn.nzcv Rn imm5 nzcv cond - imm = emitGetInsSC(id); - assert(isValidImmCondFlagsImm5(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= ((code_t)cfi.imm5 << 16); // iiiii - code |= insEncodeFlags(cfi.flags); // nzcv - code |= insEncodeCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; +/***************************************************************************** + * + * Returns the encoding to apply a Shift Type on the Rm register + */ - case IF_DI_2A: // DI_2A X.......shiiiiii iiiiiinnnnnddddd Rd Rn imm(i12,sh) - assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); - imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeShiftImm12(id->idInsOpt()); // sh - code |= ((code_t)imm << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); +/*static*/ emitter::code_t emitter::insEncodeShiftType(insOpts opt) +{ + if (emitter::insOptsNone(opt)) + { + // None implies the we encode LSL (with a zero immediate) + opt = INS_OPTS_LSL; + } + assert(emitter::insOptsAnyShift(opt)); - if (id->idIsReloc()) - { - assert(sz == sizeof(instrDesc)); - assert(id->idAddr()->iiaAddr != nullptr); - emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADD_LO12 - : IMAGE_REL_ARM64_PAGEOFFSET_12A); - } - break; + emitter::code_t option = (emitter::code_t)opt - (emitter::code_t)INS_OPTS_LSL; + assert(option <= 3); - case IF_DI_2B: // DI_2B X.........Xnnnnn ssssssnnnnnddddd Rd Rn imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // Reg2 also in mmmmm - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - dst += emitOutput_Instr(dst, code); - break; + return option << 22; // bits 23, 22 +} - case IF_DI_2C: // DI_2C X........Nrrrrrr ssssssnnnnnddddd Rd Rn imm(N,r,s) - imm = emitGetInsSC(id); - assert(isValidImmNRS(imm, id->idOpSize())); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 10); // Nrrrrrrssssss - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding to apply a 12 bit left shift to the immediate + */ - case IF_DI_2D: // DI_2D X........Nrrrrrr ssssssnnnnnddddd Rd Rn imr, imms (N,r,s) - if (ins == INS_asr || ins == INS_lsl || ins == INS_lsr) - { - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); +/*static*/ emitter::code_t emitter::insEncodeShiftImm12(insOpts opt) +{ + if (emitter::insOptsLSL12(opt)) + { + return 0x00400000; // set the bit at location 22 + } + return 0; +} - // Shift immediates are aliases of the SBFM/UBFM instructions - // that actually take 2 registers and 2 constants, - // Since we stored the shift immediate value - // we need to calculate the N,R and S values here. +/***************************************************************************** + * + * Returns the encoding to have the Rm register use an extend operation + */ - bitMaskImm bmi; - bmi.immNRS = 0; +/*static*/ emitter::code_t emitter::insEncodeExtend(insOpts opt) +{ + if (emitter::insOptsNone(opt) || (opt == INS_OPTS_LSL)) + { + // None or LSL implies the we encode UXTX + opt = INS_OPTS_UXTX; + } + assert(emitter::insOptsAnyExtend(opt)); - bmi.immN = (size == EA_8BYTE) ? 1 : 0; - bmi.immR = imm; - bmi.immS = (size == EA_8BYTE) ? 0x3f : 0x1f; + emitter::code_t option = (emitter::code_t)opt - (emitter::code_t)INS_OPTS_UXTB; + assert(option <= 7); - // immR and immS are now set correctly for INS_asr and INS_lsr - // but for INS_lsl we have to adjust the values for immR and immS - // - if (ins == INS_lsl) - { - bmi.immR = -imm & bmi.immS; - bmi.immS = bmi.immS - imm; - } + return option << 13; // bits 15,14,13 +} - // setup imm with the proper 13 bit value N:R:S - // - imm = bmi.immNRS; - } - else - { - // The other instructions have already have encoded N,R and S values - imm = emitGetInsSC(id); - } - assert(isValidImmNRS(imm, id->idOpSize())); +/***************************************************************************** + * + * Returns the encoding to scale the Rm register by {0,1,2,3,4} + * when using an extend operation + */ - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 10); // Nrrrrrrssssss - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeExtendScale(ssize_t imm) +{ + assert((imm >= 0) && (imm <= 4)); - case IF_DR_1D: // DR_1D X............... cccc.......ddddd Rd cond - imm = emitGetInsSC(id); - assert(isValidImmCond(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeInvertedCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; + return (emitter::code_t)imm << 10; // bits 12,11,10 +} - case IF_DR_2A: // DR_2A X..........mmmmm ......nnnnn..... Rn Rm - assert(insOptsNone(id->idInsOpt())); - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding to have the Rm register be auto scaled by the ld/st size + */ - case IF_DR_2B: // DR_2B X.......sh.mmmmm ssssssnnnnn..... Rn Rm {LSL,LSR,ASR,ROR} imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeShiftType(id->idInsOpt()); // sh - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeReg3Scale(bool isScaled) +{ + if (isScaled) + { + return 0x00001000; // set the bit at location 12 + } + else + { + return 0; + } +} - case IF_DR_2C: // DR_2C X..........mmmmm ooosssnnnnn..... Rn Rm ext(Rm) LSL imm(0-4) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert((imm >= 0) && (imm <= 4)); // imm [0..4] - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeExtend(id->idInsOpt()); // ooo - code |= insEncodeExtendScale(imm); // sss - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value as 9-bits at bit locations '21-16' for high and '12-10' for low. + */ - case IF_DR_2D: // DR_2D X..........nnnnn cccc..nnnnnddddd Rd Rn cond - imm = emitGetInsSC(id); - assert(isValidImmCond(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - code |= insEncodeInvertedCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; +/*static*/ emitter::code_t emitter::insEncodeSimm9h9l_21_to_16_and_12_to_10(ssize_t imm) +{ + assert(isValidSimm<9>(imm)); - case IF_DR_2E: // DR_2E X..........mmmmm ...........ddddd Rd Rm - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; + if (imm < 0) + { + imm = (imm & 0x1FF); + } - case IF_DR_2F: // DR_2F X.......sh.mmmmm ssssss.....ddddd Rd Rm {LSL,LSR,ASR} imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeShiftType(id->idInsOpt()); // sh - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; + code_t h = (code_t)(imm & 0x1F8) << 13; // encode high 6-bits at locations '21-16' + code_t l = (code_t)((imm & ~0x1F8) & 0x7) << 10; // encode low 3-bits at locations '12-10' - case IF_DR_2G: // DR_2G X............... .....xnnnnnddddd Rd Rn - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - if (ins == INS_rev) - { - if (size == EA_8BYTE) - { - code |= 0x00000400; // x - bit at location 10 - } - } - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; + return (h | l); +} - case IF_DR_2H: // DR_2H X........X...... ......nnnnnddddd Rd Rn - code = emitInsCode(ins, fmt); - code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 2 as 4-bits at bit locations '19-16'. + */ - case IF_DR_2I: // DR_2I X..........mmmmm cccc..nnnnn.nzcv Rn Rm nzcv cond - imm = emitGetInsSC(id); - assert(isValidImmCondFlags(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - code |= insEncodeFlags(cfi.flags); // nzcv - code |= insEncodeCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf2_19_to_16(ssize_t imm) +{ + assert((isValidSimm_MultipleOf<4, 2>(imm))); + return insEncodeSimm<19, 16>(imm / 2); +} - case IF_DR_3A: // DR_3A X..........mmmmm ......nnnnnmmmmm Rd Rn Rm - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - if (id->idIsLclVar()) - { - code |= insEncodeReg_Rm(codeGen->rsGetRsvdReg()); // mmmmm - } - else - { - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - } - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 3 as 4-bits at bit locations '19-16'. + */ - case IF_DR_3B: // DR_3B X.......sh.mmmmm ssssssnnnnnddddd Rd Rn Rm {LSL,LSR,ASR} imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeShiftType(id->idInsOpt()); // sh - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf3_19_to_16(ssize_t imm) +{ + assert((isValidSimm_MultipleOf<4, 3>(imm))); + return insEncodeSimm<19, 16>(imm / 3); +} - case IF_DR_3C: // DR_3C X..........mmmmm ooosssnnnnnddddd Rd Rn Rm ext(Rm) LSL imm(0-4) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert((imm >= 0) && (imm <= 4)); // imm [0..4] - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeExtend(id->idInsOpt()); // ooo - code |= insEncodeExtendScale(imm); // sss - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 4 as 4-bits at bit locations '19-16'. + */ - case IF_DR_3D: // DR_3D X..........mmmmm cccc..nnnnnddddd Rd Rn Rm cond - imm = emitGetInsSC(id); - assert(isValidImmCond(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf4_19_to_16(ssize_t imm) +{ + assert((isValidSimm_MultipleOf<4, 4>(imm))); + return insEncodeSimm<19, 16>(imm / 4); +} - case IF_DR_3E: // DR_3E X........X.mmmmm ssssssnnnnnddddd Rd Rn Rm imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 16 as 4-bits at bit locations '19-16'. + */ - case IF_DR_4A: // DR_4A X..........mmmmm .aaaaannnnnmmmmm Rd Rn Rm Ra - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeReg_Ra(id->idReg4()); // aaaaa - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf16_19_to_16(ssize_t imm) +{ + assert((isValidSimm_MultipleOf<4, 16>(imm))); + return insEncodeSimm<19, 16>(imm / 16); +} - case IF_DV_1A: // DV_1A .........X.iiiii iii........ddddd Vd imm8 (fmov - immediate scalar) - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= ((code_t)imm << 13); // iiiii iii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 32 as 4-bits at bit locations '19-16'. + */ - case IF_DV_1B: // DV_1B .QX..........iii cmod..iiiiiddddd Vd imm8 (immediate vector) - imm = emitGetInsSC(id) & 0x0ff; - immShift = (emitGetInsSC(id) & 0x700) >> 8; - elemsize = optGetElemsize(id->idInsOpt()); - cmode = 0; - switch (elemsize) - { // cmode - case EA_1BYTE: - cmode = 0xE; // 1110 - break; - case EA_2BYTE: - cmode = 0x8; - cmode |= (immShift << 1); // 10x0 - break; - case EA_4BYTE: - if (immShift < 4) - { - cmode = 0x0; - cmode |= (immShift << 1); // 0xx0 - } - else // MSL - { - cmode = 0xC; - if (immShift & 2) - cmode |= 1; // 110x - } - break; - case EA_8BYTE: - cmode = 0xE; // 1110 - break; - default: - unreached(); - break; - } +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf32_19_to_16(ssize_t imm) +{ + assert((isValidSimm_MultipleOf<4, 32>(imm))); + return insEncodeSimm<19, 16>(imm / 32); +} - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - if ((ins == INS_fmov) || (ins == INS_movi)) - { - if (elemsize == EA_8BYTE) - { - code |= 0x20000000; // X - } - } - if (ins != INS_fmov) - { - assert((cmode >= 0) && (cmode <= 0xF)); - code |= (cmode << 12); // cmod - } - code |= (((code_t)imm >> 5) << 16); // iii - code |= (((code_t)imm & 0x1f) << 5); // iiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 2 as 5-bits at bit locations '20-16'. + */ - case IF_DV_1C: // DV_1C .........X...... ......nnnnn..... Vn #0.0 (fcmp - with zero) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vn(id->idReg1()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf2_20_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<5, 2>(imm))); + return insEncodeUimm<20, 16>(imm / 2); +} - case IF_DV_2A: // DV_2A .Q.......X...... ......nnnnnddddd Vd Vn (fabs, fcvt - vector) - case IF_DV_2R: // DV_2R .Q.......X...... ......nnnnnddddd Sd Vn (fmaxnmv, fmaxv, fminnmv, fminv) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - if ((ins == INS_fcvtl) || (ins == INS_fcvtl2) || (ins == INS_fcvtn) || (ins == INS_fcvtn2)) - { - // fcvtl{2} and fcvtn{2} encode the element size as - // esize = 16 << UInt(sz) - if (elemsize == EA_4BYTE) - { - code |= 0x00400000; // X - } - else - { - assert(elemsize == EA_2BYTE); - } - } - else - { - code |= insEncodeFloatElemsize(elemsize); // X - } - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 4 as 5-bits at bit locations '20-16'. + */ - case IF_DV_2B: // DV_2B .Q.........iiiii ......nnnnnddddd Rd Vn[] (umov/smov - to general) - elemsize = id->idOpSize(); - index = emitGetInsSC(id); - datasize = (elemsize == EA_8BYTE) ? EA_16BYTE : EA_8BYTE; - if (ins == INS_smov) - { - datasize = EA_16BYTE; - } - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(datasize); // Q - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf4_20_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<5, 4>(imm))); + return insEncodeUimm<20, 16>(imm / 4); +} - case IF_DV_2C: // DV_2C .Q.........iiiii ......nnnnnddddd Vd Rn (dup/ins - vector from general) - if (ins == INS_dup) - { - datasize = id->idOpSize(); - elemsize = optGetElemsize(id->idInsOpt()); - index = 0; - } - else // INS_ins - { - datasize = EA_16BYTE; - elemsize = id->idOpSize(); - index = emitGetInsSC(id); - } - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(datasize); // Q - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 8 as 5-bits at bit locations '20-16'. + */ - case IF_DV_2D: // DV_2D .Q.........iiiii ......nnnnnddddd Vd Vn[] (dup - vector) - index = emitGetInsSC(id); - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf8_20_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<5, 8>(imm))); + return insEncodeUimm<20, 16>(imm / 8); +} - case IF_DV_2E: // DV_2E ...........iiiii ......nnnnnddddd Vd Vn[] (dup - scalar) - index = emitGetInsSC(id); - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 2 as 6-bits at bit locations '21-16'. + */ - case IF_DV_2F: // DV_2F ...........iiiii .jjjj.nnnnnddddd Vd[] Vn[] (ins - element) - elemsize = id->idOpSize(); - imm = emitGetInsSC(id); - index = (imm >> 4) & 0xf; - index2 = imm & 0xf; - code = emitInsCode(ins, fmt); - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeVectorIndex2(elemsize, index2); // jjjj - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf2_21_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<6, 2>(imm))); + return insEncodeUimm<21, 16>(imm / 2); +} - case IF_DV_2G: // DV_2G .........X...... ......nnnnnddddd Vd Vn (fmov, fcvtXX - register) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 4 as 6-bits at bit locations '21-16'. + */ - case IF_DV_2H: // DV_2H X........X...... ......nnnnnddddd Rd Vn (fmov - to general) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // X X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf4_21_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<6, 4>(imm))); + return insEncodeUimm<21, 16>(imm / 4); +} - case IF_DV_2I: // DV_2I X........X...... ......nnnnnddddd Vd Rn (fmov - from general) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // X X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 8 as 6-bits at bit locations '21-16'. + */ - case IF_DV_2J: // DV_2J ........SS.....D D.....nnnnnddddd Vd Vn (fcvt) - code = emitInsCode(ins, fmt); - code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // SS DD - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf8_21_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<6, 8>(imm))); + return insEncodeUimm<21, 16>(imm / 8); +} - case IF_DV_2K: // DV_2K .........X.mmmmm ......nnnnn..... Vn Vm (fcmp) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vn(id->idReg1()); // nnnnn - code |= insEncodeReg_Vm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value as 1-bit at bit locations '23'. + */ - case IF_DV_2L: // DV_2L ........XX...... ......nnnnnddddd Vd Vn (abs, neg - scalar) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm1_23(ssize_t imm) +{ + assert(isValidUimm<1>(imm)); + return (code_t)imm << 23; +} - case IF_DV_2M: // DV_2M .Q......XX...... ......nnnnnddddd Vd Vn (abs, neg - vector) - case IF_DV_2T: // DV_2T .Q......XX...... ......nnnnnddddd Sd Vn (addv, saddlv, smaxv, sminv, uaddlv, - // umaxv, uminv) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value as 3-bits at bit locations '23-22' for high and '12' for low. + */ + +/*static*/ emitter::code_t emitter::insEncodeUimm3h3l_23_to_22_and_12(ssize_t imm) +{ + assert(isValidUimm<3>(imm)); + + code_t h = (code_t)(imm & 0x6) << 21; // encode high 2-bits at locations '23-22' + code_t l = (code_t)(imm & 0x1) << 12; // encode low 1-bit at locations '12' + + return (h | l); +} - case IF_DV_2N: // DV_2N .........iiiiiii ......nnnnnddddd Vd Vn imm (shift - scalar) - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorShift(elemsize, emitInsIsVectorRightShift(ins), imm); // iiiiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value as 4-bits starting from 1, at bit locations '19-16'. + */ - case IF_DV_2O: // DV_2O .Q.......iiiiiii ......nnnnnddddd Vd Vn imm (shift - vector) - imm = emitGetInsSC(id); - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeVectorShift(elemsize, emitInsIsVectorRightShift(ins), imm); // iiiiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm4From1_19_to_16(ssize_t imm) +{ + assert(isValidUimmFrom1<4>(imm)); + return (code_t)(imm - 1) << 16; +} - case IF_DV_2P: // DV_2P ............... ......nnnnnddddd Vd Vn (aes*, sha1su1) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value as 8-bits at bit locations '12-5'. + */ - case IF_DV_2Q: // DV_2Q .........X...... ......nnnnnddddd Vd Vn (faddp, fmaxnmp, fmaxp, fminnmp, - // fminp - scalar) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +/*static*/ emitter::code_t emitter::insEncodeImm8_12_to_5(ssize_t imm) +{ + assert(isValidSimm<8>(imm) || isValidUimm<8>(imm)); + return (code_t)((imm & 0xFF) << 5); +} - case IF_DV_2S: // DV_2S ........XX...... ......nnnnnddddd Sd Vn (addp - scalar) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +BYTE* emitter::emitOutputLoadLabel(BYTE* dst, BYTE* srcAddr, BYTE* dstAddr, instrDescJmp* id) +{ + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); + regNumber dstReg = id->idReg1(); + if (id->idjShort) + { + // adr x, [rel addr] -- compute address: current addr(ip) + rel addr. + assert(ins == INS_adr); + assert(fmt == IF_DI_1E); + ssize_t distVal = (ssize_t)(dstAddr - srcAddr); + dst = emitOutputShortAddress(dst, ins, fmt, distVal, dstReg); + } + else + { + // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr + assert(fmt == IF_LARGEADR); + ssize_t relPageAddr = computeRelPageAddr((size_t)dstAddr, (size_t)srcAddr); + dst = emitOutputShortAddress(dst, INS_adrp, IF_DI_1E, relPageAddr, dstReg); - case IF_DV_2U: // DV_2U ................ ......nnnnnddddd Sd Sn (sha1h) - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; + // add x, x, page offs -- compute address = page addr + page offs + ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits + assert(isValidUimm<12>(imm12)); + code_t code = + emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) + code |= insEncodeDatasize(EA_8BYTE); // X + code |= ((code_t)imm12 << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rd(dstReg); // ddddd + code |= insEncodeReg_Rn(dstReg); // nnnnn + dst += emitOutput_Instr(dst, code); + } + return dst; +} - case IF_DV_3A: // DV_3A .Q......XX.mmmmm ......nnnnnddddd Vd Vn Vm (vector) - code = emitInsCode(ins, fmt); - elemsize = optGetElemsize(id->idInsOpt()); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Output a local jump or other instruction with a pc-relative immediate. + * Note that this may be invoked to overwrite an existing jump instruction at 'dst' + * to handle forward branch patching. + */ - case IF_DV_3AI: // DV_3AI .Q......XXLMmmmm ....H.nnnnnddddd Vd Vn Vm[] (vector) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - elemsize = optGetElemsize(id->idInsOpt()); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeVectorIndexLMH(elemsize, imm); // LM H - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; +BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) +{ + instrDescJmp* id = (instrDescJmp*)i; - case IF_DV_3B: // DV_3B .Q.......X.mmmmm ......nnnnnddddd Vd Vn Vm (vector) - code = emitInsCode(ins, fmt); - elemsize = optGetElemsize(id->idInsOpt()); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; + unsigned srcOffs; + unsigned dstOffs; + BYTE* srcAddr; + BYTE* dstAddr; + ssize_t distVal; - case IF_DV_3BI: // DV_3BI .Q.......XLmmmmm ....H.nnnnnddddd Vd Vn Vm[] (vector by element) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - elemsize = optGetElemsize(id->idInsOpt()); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeFloatIndex(elemsize, imm); // L H - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; + // Set default ins/fmt from id. + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); - case IF_DV_3C: // DV_3C .Q.........mmmmm ......nnnnnddddd Vd Vn Vm (vector) - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; + bool loadLabel = false; + bool isJump = false; + bool loadConstant = false; - case IF_DV_3D: // DV_3D .........X.mmmmm ......nnnnnddddd Vd Vn Vm (scalar) - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(id->idOpSize()); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); + switch (ins) + { + default: + isJump = true; break; - case IF_DV_3DI: // DV_3DI .........XLmmmmm ....H.nnnnnddddd Vd Vn Vm[] (scalar by element) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeFloatIndex(elemsize, imm); // L H - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); + case INS_tbz: + case INS_tbnz: + case INS_cbz: + case INS_cbnz: + isJump = true; break; - case IF_DV_3E: // DV_3E ........XX.mmmmm ......nnnnnddddd Vd Vn Vm (scalar) - code = emitInsCode(ins, fmt); - elemsize = id->idOpSize(); - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); + case INS_ldr: + case INS_ldrsw: + loadConstant = true; break; - case IF_DV_3EI: // DV_3EI ........XXLMmmmm ....H.nnnnnddddd Vd Vn Vm[] (scalar by element) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeVectorIndexLMH(elemsize, imm); // LM H - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); + case INS_adr: + case INS_adrp: + loadLabel = true; break; + } - case IF_DV_3F: // DV_3F ...........mmmmm ......nnnnnddddd Vd Vn Vm (vector) - source dest regs overlap - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; + /* Figure out the distance to the target */ - case IF_DV_3G: // DV_3G .Q.........mmmmm .iiii.nnnnnddddd Vd Vn Vm imm (vector) - imm = emitGetInsSC(id); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - code |= ((code_t)imm << 11); // iiii - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; + srcOffs = emitCurCodeOffs(dst); + srcAddr = emitOffsetToPtr(srcOffs); + + if (id->idAddr()->iiaIsJitDataOffset()) + { + assert(loadConstant || loadLabel); + int doff = id->idAddr()->iiaGetJitDataOffset(); + assert(doff >= 0); + ssize_t imm = emitGetInsSC(id); + assert((imm >= 0) && (imm < 0x1000)); // 0x1000 is arbitrary, currently 'imm' is always 0 + + unsigned dataOffs = (unsigned)(doff + imm); + assert(dataOffs < emitDataSize()); + dstAddr = emitDataOffsetToPtr(dataOffs); - case IF_DV_4A: // DV_4A .........X.mmmmm .aaaaannnnnddddd Vd Va Vn Vm (scalar) - code = emitInsCode(ins, fmt); - elemsize = id->idOpSize(); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - code |= insEncodeReg_Va(id->idReg4()); // aaaaa - dst += emitOutput_Instr(dst, code); - break; + regNumber dstReg = id->idReg1(); + regNumber addrReg = dstReg; // an integer register to compute long address. + emitAttr opSize = id->idOpSize(); - case IF_SN_0A: // SN_0A ................ ................ + if (loadConstant) { - bool skipIns = false; -#if FEATURE_LOOP_ALIGN - if (id->idIns() == INS_align) + if (id->idjShort) { - // IG can be marked as not needing alignment after emitting align instruction. - // Alternatively, there are fewer align instructions needed than emitted. - // If that is the case, skip outputting alignment. - if (!ig->endsWithAlignInstr() || id->idIsEmptyAlign()) + // ldr x/v, [rel addr] -- load constant from current addr(ip) + rel addr. + assert(ins == INS_ldr); + assert(fmt == IF_LS_1A); + distVal = (ssize_t)(dstAddr - srcAddr); + dst = emitOutputShortConstant(dst, ins, fmt, distVal, dstReg, opSize); + } + else + { + // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr + assert(fmt == IF_LARGELDC); + ssize_t relPageAddr = computeRelPageAddr((size_t)dstAddr, (size_t)srcAddr); + if (isVectorRegister(dstReg)) { - skipIns = true; - } + // Update addrReg with the reserved integer register + // since we cannot use dstReg (vector) to load constant directly from memory. -#ifdef DEBUG - if (!ig->endsWithAlignInstr()) - { - // Validate if the state is correctly updated - assert(id->idIsEmptyAlign()); + // If loading a 16-byte value, we will need to load directly into dstReg. + // Thus, encode addrReg for the ld1 instruction. + if (opSize == EA_16BYTE) + { + addrReg = encodingSPtoZR(id->idReg2()); + } + else + { + addrReg = id->idReg2(); + } + + assert(isGeneralRegister(addrReg)); } -#endif - sz = sizeof(instrDescAlign); - ins = INS_nop; -#ifdef DEBUG - // Under STRESS_EMITTER, if this is the 'align' before the 'jmp' instruction, - // then add "bkpt" instruction. - instrDescAlign* alignInstr = (instrDescAlign*)id; + ins = INS_adrp; + fmt = IF_DI_1E; + dst = emitOutputShortAddress(dst, ins, fmt, relPageAddr, addrReg); - if (emitComp->compStressCompile(Compiler::STRESS_EMITTER, 50) && alignInstr->isPlacedAfterJmp && - !skipIns) + ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits + assert(isValidUimm<12>(imm12)); + + // Special case: emit add + ld1 instructions for loading 16-byte data into vector register. + if (isVectorRegister(dstReg) && (opSize == EA_16BYTE)) { - // There is no good way to squeeze in "bkpt" as well as display it - // in the disassembly because there is no corresponding instrDesc for - // it. As such, leave it as is, the "0xD43E0000" bytecode will be seen - // next to the nop instruction in disasm. - // e.g. D43E0000 align [4 bytes for IG07] - ins = INS_BREAKPOINT; - fmt = IF_SI_0A; + const emitAttr elemSize = EA_1BYTE; + const insOpts opt = optMakeArrangement(opSize, elemSize); + + assert(isGeneralRegisterOrSP(addrReg)); + assert(isValidVectorElemsize(elemSize)); + assert(isValidArrangement(opSize, opt)); + + // Calculate page addr + page offs, then emit ld1 instruction. + dst = emitOutputVectorConstant(dst, imm12, dstReg, addrReg, opSize, elemSize); } -#endif - } -#endif // FEATURE_LOOP_ALIGN + else + { + // ldr x, [x, 0] -- load constant from address into integer register. + ins = INS_ldr; + fmt = IF_LS_2B; + dst = emitOutputShortConstant(dst, ins, fmt, imm12, addrReg, opSize); - if (!skipIns) - { - code = emitInsCode(ins, fmt); - dst += emitOutput_Instr(dst, code); - } + // fmov v, d -- copy constant in integer register to vector register. + // This is needed only for vector constant. + if (addrReg != dstReg) + { + // fmov Vd,Rn DV_2I X00111100X100111 000000nnnnnddddd 1E27 0000 Vd,Rn + // (scalar, from general) + assert(isVectorRegister(dstReg) && isGeneralRegister(addrReg)); + ins = INS_fmov; + fmt = IF_DV_2I; + code_t code = emitInsCode(ins, fmt); - break; + code |= insEncodeReg_Vd(dstReg); // ddddd + code |= insEncodeReg_Rn(addrReg); // nnnnn + if (id->idOpSize() == EA_8BYTE) + { + code |= 0x80400000; // X ... X + } + dst += emitOutput_Instr(dst, code); + } + } + } + } + else + { + assert(loadLabel); + dst = emitOutputLoadLabel(dst, srcAddr, dstAddr, id); } - case IF_SI_0A: // SI_0A ...........iiiii iiiiiiiiiii..... imm16 - imm = emitGetInsSC(id); - assert(isValidUimm<16>(imm)); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 5); // iiiii iiiiiiiiiii - dst += emitOutput_Instr(dst, code); - break; + return dst; + } - case IF_SI_0B: // SI_0B ................ ....bbbb........ imm4 - barrier - imm = emitGetInsSC(id); - assert((imm >= 0) && (imm <= 15)); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 8); // bbbb - dst += emitOutput_Instr(dst, code); - break; + assert(loadLabel || isJump); - case IF_SR_1A: // SR_1A ................ ...........ttttt Rt (dc zva, mrs) - assert(insOptsNone(id->idInsOpt())); - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - dst += emitOutput_Instr(dst, code); - break; + if (id->idAddr()->iiaHasInstrCount()) + { + assert(ig != NULL); + int instrCount = id->idAddr()->iiaGetInstrCount(); + unsigned insNum = emitFindInsNum(ig, id); + if (instrCount < 0) + { + // Backward branches using instruction count must be within the same instruction group. + assert(insNum + 1 >= (unsigned)(-instrCount)); + } - default: - dst = emitOutput_InstrSve(dst, id); - break; + dstOffs = ig->igOffs + emitFindOffset(ig, (insNum + 1 + instrCount)); + dstAddr = emitOffsetToPtr(dstOffs); + } + else + { + dstOffs = id->idAddr()->iiaIGlabel->igOffs; + dstAddr = emitOffsetToPtr(dstOffs); } - // Determine if any registers now hold GC refs, or whether a register that was overwritten held a GC ref. - // We assume here that "id->idGCref()" is not GC_NONE only if the instruction described by "id" writes a - // GC ref to register "id->idReg1()". (It may, apparently, also not be GC_NONE in other cases, such as - // for stores, but we ignore those cases here.) - if (emitInsMayWriteToGCReg(id)) // True if "id->idIns()" writes to a register than can hold GC ref. + distVal = (ssize_t)(dstAddr - srcAddr); + + if (dstOffs <= srcOffs) { - // We assume that "idReg1" is the primary destination register for all instructions - assert(!emitInsDestIsOp2(ins)); - if (id->idGCref() != GCT_NONE) +#if DEBUG_EMIT + /* This is a backward jump - distance is known at this point */ + + if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) { - emitGCregLiveUpd(id->idGCref(), id->idReg1(), dst); + size_t blkOffs = id->idjIG->igOffs; + + if (INTERESTING_JUMP_NUM == 0) + printf("[3] Jump %u:\n", id->idDebugOnlyInfo()->idNum); + printf("[3] Jump block is at %08X - %02X = %08X\n", blkOffs, emitOffsAdj, blkOffs - emitOffsAdj); + printf("[3] Jump is at %08X - %02X = %08X\n", srcOffs, emitOffsAdj, srcOffs - emitOffsAdj); + printf("[3] Label block is at %08X - %02X = %08X\n", dstOffs, emitOffsAdj, dstOffs - emitOffsAdj); } - else +#endif + } + else + { + /* This is a forward jump - distance will be an upper limit */ + + emitFwdJumps = true; + + /* The target offset will be closer by at least 'emitOffsAdj', but only if this + jump doesn't cross the hot-cold boundary. */ + + if (!emitJumpCrossHotColdBoundary(srcOffs, dstOffs)) { - emitGCregDeadUpd(id->idReg1(), dst); + dstOffs -= emitOffsAdj; + distVal -= emitOffsAdj; } - if (emitInsMayWriteMultipleRegs(id)) + /* Record the location of the jump for later patching */ + + id->idjOffs = dstOffs; + + /* Are we overflowing the id->idjOffs bitfield? */ + if (id->idjOffs != dstOffs) + IMPL_LIMITATION("Method is too large"); + +#if DEBUG_EMIT + if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) { - // INS_ldp etc... - // "idReg2" is the secondary destination register - if (id->idGCrefReg2() != GCT_NONE) - { - emitGCregLiveUpd(id->idGCrefReg2(), id->idReg2(), dst); - } - else - { - emitGCregDeadUpd(id->idReg2(), dst); - } + size_t blkOffs = id->idjIG->igOffs; + + if (INTERESTING_JUMP_NUM == 0) + printf("[4] Jump %u:\n", id->idDebugOnlyInfo()->idNum); + printf("[4] Jump block is at %08X\n", blkOffs); + printf("[4] Jump is at %08X\n", srcOffs); + printf("[4] Label block is at %08X - %02X = %08X\n", dstOffs + emitOffsAdj, emitOffsAdj, dstOffs); } +#endif + } + +#ifdef DEBUG + if (0 && emitComp->verbose) + { + size_t sz = 4; + int distValSize = id->idjShort ? 4 : 8; + printf("; %s jump [%08X/%03u] from %0*X to %0*X: dist = 0x%08X\n", (dstOffs <= srcOffs) ? "Fwd" : "Bwd", + dspPtr(id), id->idDebugOnlyInfo()->idNum, distValSize, srcOffs + sz, distValSize, dstOffs, distVal); } +#endif + + /* For forward jumps, record the address of the distance value */ + id->idjTemp.idjAddr = (distVal > 0) ? dst : NULL; + + assert(insOptsNone(id->idInsOpt())); -SKIP_GC_UPDATE: - // Now we determine if the instruction has written to a (local variable) stack location, and either written a GC - // ref or overwritten one. - if (emitInsWritesToLclVarStackLoc(id) || emitInsWritesToLclVarStackLocPair(id)) + if (isJump) { - int varNum = id->idAddr()->iiaLclVar.lvaVarNum(); - unsigned ofs = AlignDown(id->idAddr()->iiaLclVar.lvaOffset(), TARGET_POINTER_SIZE); - bool FPbased; - int adr = emitComp->lvaFrameAddress(varNum, &FPbased); - if (id->idGCref() != GCT_NONE) + if (id->idjShort) { - emitGCvarLiveUpd(adr + ofs, varNum, id->idGCref(), dst DEBUG_ARG(varNum)); + // Short conditional/unconditional jump + assert(!id->idjKeepLong); + assert(emitJumpCrossHotColdBoundary(srcOffs, dstOffs) == false); + assert((fmt == IF_BI_0A) || (fmt == IF_BI_0B) || (fmt == IF_BI_1A) || (fmt == IF_BI_1B)); + dst = emitOutputShortBranch(dst, ins, fmt, distVal, id); } else { - // If the type of the local is a gc ref type, update the liveness. - var_types vt; - if (varNum >= 0) - { - // "Regular" (non-spill-temp) local. - vt = var_types(emitComp->lvaTable[varNum].lvType); - } - else - { - TempDsc* tmpDsc = codeGen->regSet.tmpFindNum(varNum); - vt = tmpDsc->tdTempType(); - } - if (vt == TYP_REF || vt == TYP_BYREF) - { - emitGCvarDeadUpd(adr + ofs, dst DEBUG_ARG(varNum)); - } - } - if (emitInsWritesToLclVarStackLocPair(id)) - { - int varNum2 = varNum; - int adr2 = adr; - unsigned ofs2 = ofs; - unsigned ofs2Dist; + // Long conditional/unconditional jump - if (id->idIsLclVarPair()) + if (fmt == IF_LARGEJMP) { - bool FPbased2; + // This is a pseudo-instruction format representing a large conditional branch, to allow + // us to get a greater branch target range than we can get by using a straightforward conditional + // branch. It is encoded as a short conditional branch that branches around a long unconditional + // branch. + // + // Conceptually, we have: + // + // b L_target + // + // The code we emit is: + // + // b L_not // 4 bytes. Note that we reverse the condition. + // b L_target // 4 bytes + // L_not: + // + // Note that we don't actually insert any blocks: we simply encode "b L_not" as a branch with + // the correct offset. Note also that this works for both integer and floating-point conditions, because + // the condition inversion takes ordered/unordered into account, preserving NaN behavior. For example, + // "GT" (greater than) is inverted to "LE" (less than, equal, or unordered). - emitLclVarAddr* lclVarAddr2 = emitGetLclVarPairLclVar2(id); - varNum2 = lclVarAddr2->lvaVarNum(); - ofs2 = lclVarAddr2->lvaOffset(); + instruction reverseIns; + insFormat reverseFmt; - // If there are 2 GC vars in this instrDesc, get the 2nd variable - // that should be tracked. - adr2 = emitComp->lvaFrameAddress(varNum2, &FPbased2); - ofs2Dist = EA_SIZE_IN_BYTES(size); -#ifdef DEBUG - assert(FPbased == FPbased2); - if (FPbased) - { - assert(id->idReg3() == REG_FP); - } - else + switch (ins) { - assert(id->idReg3() == REG_SP); + case INS_cbz: + reverseIns = INS_cbnz; + reverseFmt = IF_BI_1A; + break; + case INS_cbnz: + reverseIns = INS_cbz; + reverseFmt = IF_BI_1A; + break; + case INS_tbz: + reverseIns = INS_tbnz; + reverseFmt = IF_BI_1B; + break; + case INS_tbnz: + reverseIns = INS_tbz; + reverseFmt = IF_BI_1B; + break; + default: + reverseIns = emitJumpKindToIns(emitReverseJumpKind(emitInsToJumpKind(ins))); + reverseFmt = IF_BI_0B; } - assert(varNum2 != -1); -#endif // DEBUG - } - else - { - ofs2Dist = TARGET_POINTER_SIZE; - ofs2 += ofs2Dist; + + dst = emitOutputShortBranch(dst, + reverseIns, // reverse the conditional instruction + reverseFmt, 8, /* 8 bytes from start of this large conditional + pseudo-instruction to L_not. */ + id); + + // Now, pretend we've got a normal unconditional branch, and fall through to the code to emit that. + ins = INS_b; + fmt = IF_BI_0A; + + // The distVal was computed based on the beginning of the pseudo-instruction, + // So subtract the size of the conditional branch so that it is relative to the + // unconditional branch. + distVal -= 4; } - ofs2 = AlignDown(ofs2, ofs2Dist); + assert(fmt == IF_BI_0A); + assert((distVal & 1) == 0); + code_t code = emitInsCode(ins, fmt); + const bool doRecordRelocation = emitComp->opts.compReloc && emitJumpCrossHotColdBoundary(srcOffs, dstOffs); - if (id->idGCrefReg2() != GCT_NONE) + if (doRecordRelocation) { -#ifdef DEBUG - if (id->idGCref() != GCT_NONE) - { - // If 1st register was a gc-var, then make sure the offset - // are correctly set for the 2nd register that is holding - // another gc-var. - assert((adr + ofs + ofs2Dist) == (adr2 + ofs2)); - } -#endif - emitGCvarLiveUpd(adr2 + ofs2, varNum2, id->idGCrefReg2(), dst DEBUG_ARG(varNum2)); + // dst isn't an actual final target location, just some intermediate + // location. Thus we cannot make any guarantees about distVal (not + // even the direction/sign). Instead we don't encode any offset and + // rely on the relocation to do all the work } else { - // If the type of the local is a gc ref type, update the liveness. - var_types vt; - if (varNum2 >= 0) - { - // "Regular" (non-spill-temp) local. - vt = var_types(emitComp->lvaTable[varNum2].lvType); - } - else - { - TempDsc* tmpDsc = codeGen->regSet.tmpFindNum(varNum2); - vt = tmpDsc->tdTempType(); - } - if (vt == TYP_REF || vt == TYP_BYREF) + // Branch offset encodings are scaled by 4. + noway_assert((distVal & 3) == 0); + distVal >>= 2; + noway_assert(isValidSimm<26>(distVal)); + + // Insert offset into unconditional branch instruction + distVal &= 0x3FFFFFFLL; + code |= distVal; + } + + const unsigned instrSize = emitOutput_Instr(dst, code); + + if (doRecordRelocation) + { + assert(id->idjKeepLong); + if (emitComp->info.compMatchedVM) { - emitGCvarDeadUpd(adr2 + ofs2, dst DEBUG_ARG(varNum2)); + void* target = emitOffsetToPtr(dstOffs); + emitRecordRelocation((void*)dst, target, IMAGE_REL_ARM64_BRANCH26); } } + + dst += instrSize; } } + else if (loadLabel) + { + dst = emitOutputLoadLabel(dst, srcAddr, dstAddr, id); + } -#ifdef DEBUG - /* Make sure we set the instruction descriptor size correctly */ + return dst; +} - size_t expected = emitSizeOfInsDsc(id); - assert(sz == expected); +/***************************************************************************** +* +* Output a short branch instruction. +*/ +BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, instrDescJmp* id) +{ + code_t code = emitInsCode(ins, fmt); - if (emitComp->opts.disAsm || emitComp->verbose) + ssize_t loBits = (distVal & 3); + noway_assert(loBits == 0); + distVal >>= 2; // branch offset encodings are scaled by 4. + + if (fmt == IF_BI_0A) { - emitDispIns(id, false, dspOffs, true, emitCurCodeOffs(odst), *dp, (dst - *dp), ig); + // INS_b or INS_bl_local + noway_assert(isValidSimm<26>(distVal)); + distVal &= 0x3FFFFFFLL; + code |= distVal; + } + else if (fmt == IF_BI_0B) // BI_0B 01010100iiiiiiii iiiiiiiiiiiXXXXX simm19:00 + { + // INS_beq, INS_bne, etc... + noway_assert(isValidSimm<19>(distVal)); + distVal &= 0x7FFFFLL; + code |= distVal << 5; + } + else if (fmt == IF_BI_1A) // BI_1A X.......iiiiiiii iiiiiiiiiiittttt Rt simm19:00 + { + // INS_cbz or INS_cbnz + assert(id != nullptr); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + + noway_assert(isValidSimm<19>(distVal)); + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; } + else if (fmt == IF_BI_1B) // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 + { + // INS_tbz or INS_tbnz + assert(id != nullptr); + ssize_t imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + + if (imm & 0x20) // test bit 32-63 ? + { + code |= 0x80000000; // B + } + code |= ((imm & 0x1F) << 19); // bbbbb + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + + noway_assert(isValidSimm<14>(distVal)); + distVal &= 0x3FFFLL; // 14 bits + code |= distVal << 5; + } + else + { + assert(!"Unknown fmt for emitOutputShortBranch"); + } + + dst += emitOutput_Instr(dst, code); + + return dst; +} + +/***************************************************************************** +* +* Output a short address instruction. +*/ +BYTE* emitter::emitOutputShortAddress(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, regNumber reg) +{ + ssize_t loBits = (distVal & 3); + distVal >>= 2; - if (emitComp->compDebugBreak) + code_t code = emitInsCode(ins, fmt); + if (fmt == IF_DI_1E) // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 { - // For example, set JitBreakEmitOutputInstr=a6 will break when this method is called for - // emitting instruction a6, (i.e. IN00a6 in jitdump). - if ((unsigned)JitConfig.JitBreakEmitOutputInstr() == id->idDebugOnlyInfo()->idNum) - { - assert(!"JitBreakEmitOutputInstr reached"); - } - } + // INS_adr or INS_adrp + code |= insEncodeReg_Rd(reg); // ddddd - // Output any delta in GC info. - if (EMIT_GC_VERBOSE || emitComp->opts.disasmWithGC) - { - emitDispGCInfoDelta(); + noway_assert(isValidSimm<19>(distVal)); + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; + code |= loBits << 29; // 2 bits } -#else - if (emitComp->opts.disAsm) + else { - size_t expected = emitSizeOfInsDsc(id); - assert(sz == expected); - emitDispIns(id, false, 0, true, emitCurCodeOffs(odst), *dp, (dst - *dp), ig); + assert(!"Unknown fmt for emitOutputShortAddress"); } -#endif - /* All instructions are expected to generate code */ - - assert(*dp != dst || id->idIsEmptyAlign()); - - *dp = dst; + dst += emitOutput_Instr(dst, code); - return sz; + return dst; } /***************************************************************************** - * - * Append the machine code corresponding to the given SVE instruction descriptor. - */ -BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) +* +* Output a short constant instruction. +*/ +BYTE* emitter::emitOutputShortConstant( + BYTE* dst, instruction ins, insFormat fmt, ssize_t imm, regNumber reg, emitAttr opSize) { - code_t code = 0; - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - emitAttr size = id->idOpSize(); - - ssize_t imm; + code_t code = emitInsCode(ins, fmt); - switch (fmt) + if (fmt == IF_LS_1A) { - // Scalable. - case IF_SVE_AA_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise logical operations (predicated) - case IF_SVE_AB_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) - case IF_SVE_AC_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer divide vectors (predicated) - case IF_SVE_AD_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer min/max/difference (predicated) - case IF_SVE_AE_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer multiply vectors (predicated) - case IF_SVE_AF_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (predicated) - case IF_SVE_AG_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (quadwords) - case IF_SVE_AI_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (predicated) - case IF_SVE_AJ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (quadwords) - case IF_SVE_AK_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer min/max reduction (predicated) - case IF_SVE_AL_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer min/max reduction (quadwords) - case IF_SVE_AN_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise shift by vector (predicated) - case IF_SVE_AO_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise shift by wide elements (predicated) - case IF_SVE_AP_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise unary operations (predicated) - case IF_SVE_AQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer unary operations (predicated) - case IF_SVE_CL_3A: // ........xx...... ...gggnnnnnddddd -- SVE compress active elements - case IF_SVE_CM_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally broadcast element to vector - case IF_SVE_CN_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally extract element to SIMD&FP scalar - case IF_SVE_CP_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy SIMD&FP scalar register to vector - // (predicated) - case IF_SVE_CR_3A: // ........xx...... ...gggnnnnnddddd -- SVE extract element to SIMD&FP scalar register - case IF_SVE_CU_3A: // ........xx...... ...gggnnnnnddddd -- SVE reverse within elements - case IF_SVE_EP_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 integer halving add/subtract (predicated) - case IF_SVE_EQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer pairwise add and accumulate long - case IF_SVE_ER_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 integer pairwise arithmetic - case IF_SVE_ES_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer unary operations (predicated) - case IF_SVE_ET_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating add/subtract - case IF_SVE_EU_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating/rounding bitwise shift left - // (predicated) - case IF_SVE_GR_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 floating-point pairwise operations - case IF_SVE_GS_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction (quadwords) - case IF_SVE_HE_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction - case IF_SVE_HJ_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point serial reduction (predicated) - case IF_SVE_HL_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) - case IF_SVE_HQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point round to integral value - case IF_SVE_HR_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point unary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm or nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt simm21 + // INS_ldr or INS_ldrsw (PC-Relative) - case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) - case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; + ssize_t loBits = (imm & 3); + noway_assert(loBits == 0); + ssize_t distVal = imm >> 2; // load offset encodings are scaled by 4. - // Scalable with Merge or Zero predicate - case IF_SVE_AH_3A: // ........xx.....M ...gggnnnnnddddd -- SVE constructive prefix (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // nnnnn - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // ddddd - code |= insEncodePredQualifier_16(id->idPredicateReg2Merge()); // M - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + noway_assert(isValidSimm<19>(distVal)); - // Scalable with shift immediate - case IF_SVE_AM_2A: // ........xx...... ...gggxxiiiddddd -- SVE bitwise shift by immediate (predicated) + // Is the target a vector register? + if (isVectorRegister(reg)) { - bool isRightShift = emitInsIsVectorRightShift(ins); - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= - insEncodeSveShift_23_to_22_9_to_0(optGetSveElemsize(id->idInsOpt()), isRightShift, imm); // xx, xxiii - dst += emitOutput_Instr(dst, code); + code |= insEncodeDatasizeVLS(code, opSize); // XX V + code |= insEncodeReg_Vt(reg); // ttttt } - break; + else + { + assert(isGeneralRegister(reg)); + // insEncodeDatasizeLS is not quite right for this case. + // So just specialize it. + if ((ins == INS_ldr) && (opSize == EA_8BYTE)) + { + // set the operation size in bit 30 + code |= 0x40000000; + } - // Scalable, 4 regs. Reg4 in mmmmm. - case IF_SVE_AR_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE integer multiply-accumulate writing addend - // (predicated) - case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) - case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + code |= insEncodeReg_Rt(reg); // ttttt + } - // Scalable, 4 regs. Reg4 in aaaaa. - case IF_SVE_AS_4A: // ........xx.mmmmm ...gggaaaaaddddd -- SVE integer multiply-add writing multiplicand - // (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeReg_V_9_to_5(id->idReg4()); // aaaaa - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; + } + else if (fmt == IF_LS_2B) + { + // ldr Rt,[Xn+pimm12] LS_2B 1X11100101iiiiii iiiiiinnnnnttttt B940 0000 imm(0-4095<<{2,3}) + // INS_ldr or INS_ldrsw (PC-Relative) + noway_assert(isValidUimm<12>(imm)); + assert(isGeneralRegister(reg)); - // Scalable, 3 regs, no predicates - case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high - // (unpredicated) - case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) - case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient - case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_BZ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_BZ_3A_A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads - case IF_SVE_EH_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer dot product (unpredicated) - case IF_SVE_EL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply-add long - case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high - case IF_SVE_EN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add interleaved long - case IF_SVE_EO_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add long - case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp - case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) - case IF_SVE_FL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long - case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide - case IF_SVE_FN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved - case IF_SVE_FQ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise permute - case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long - case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate - case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long - case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part - case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) - case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + if (opSize == EA_8BYTE) + { + // insEncodeDatasizeLS is not quite right for this case. + // So just specialize it. + if (ins == INS_ldr) + { + // set the operation size in bit 30 + code |= 0x40000000; + } + // Low 3 bits should be 0 -- 8 byte JIT data should be aligned on 8 byte. + assert((imm & 7) == 0); + imm >>= 3; + } + else + { + assert(opSize == EA_4BYTE); + // Low 2 bits should be 0 -- 4 byte aligned data. + assert((imm & 3) == 0); + imm >>= 2; + } - // Scalable, 3 regs, no predicates. General purpose source registers - case IF_SVE_BA_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE index generation (register start, register - // increment) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + code |= insEncodeReg_Rt(reg); // ttttt + code |= insEncodeReg_Rn(reg); // nnnnn + code |= imm << 10; + } + else + { + assert(!"Unknown fmt for emitOutputShortConstant"); + } - case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // hh - code |= insEncodeUimm<22, 22>(id->idInsOpt() == INS_OPTS_SCALABLE_D ? 1 : 0); - dst += emitOutput_Instr(dst, code); - break; + dst += emitOutput_Instr(dst, code); - case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // hh - dst += emitOutput_Instr(dst, code); - break; + return dst; +} - // Immediate and pattern to general purpose. - case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count - case IF_SVE_BM_1A: // ............iiii ......pppppddddd -- SVE inc/dec register by element count - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeUimm4From1_19_to_16(imm); // iiii - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Output instructions to load a constant into a vector register. + */ +BYTE* emitter::emitOutputVectorConstant( + BYTE* dst, ssize_t imm, regNumber dstReg, regNumber addrReg, emitAttr opSize, emitAttr elemSize) +{ + // add addrReg, addrReg, page offs -- compute address = page addr + page offs. + code_t code = emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) + code |= insEncodeDatasize(EA_8BYTE); // X - use EA_8BYTE, as we are calculating 64-bit address + code |= ((code_t)imm << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rd(addrReg); // ddddd + code |= insEncodeReg_Rn(addrReg); // nnnnn + dst += emitOutput_Instr(dst, code); - case IF_SVE_BO_1A: // ...........Xiiii ......pppppddddd -- SVE saturating inc/dec register by element count - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeUimm4From1_19_to_16(imm); // iiii - code |= insEncodeSveElemsize_sz_20(id->idOpSize()); // X - dst += emitOutput_Instr(dst, code); - break; + // ld1 dstReg, addrReg -- load constant at address in addrReg into dstReg. + code = emitInsCode(INS_ld1, IF_LS_2D); // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn + code |= insEncodeVectorsize(opSize); // Q + code |= insEncodeVLSElemsize(elemSize); // ss + code |= insEncodeReg_Rn(addrReg); // nnnnn + code |= insEncodeReg_Vt(dstReg); // ttttt + dst += emitOutput_Instr(dst, code); + + return dst; +} + +/***************************************************************************** + * + * Output a call instruction. + */ - case IF_SVE_BQ_2A: // ...........iiiii ...iiinnnnnddddd -- SVE extract vector (immediate offset, destructive) - case IF_SVE_BQ_2B: // ...........iiiii ...iiimmmmmddddd -- SVE extract vector (immediate offset, destructive) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn/mmmmm - code |= insEncodeUimm<12, 10>(imm & 0b111); // iii - code |= insEncodeUimm<20, 16>(imm >> 3); // iiiii - dst += emitOutput_Instr(dst, code); - break; +unsigned emitter::emitOutputCall(insGroup* ig, BYTE* dst, instrDesc* id, code_t code) +{ + const unsigned char callInstrSize = sizeof(code_t); // 4 bytes + regMaskTP gcrefRegs; + regMaskTP byrefRegs; - case IF_SVE_BN_1A: // ............iiii ......pppppddddd -- SVE inc/dec vector by element count - case IF_SVE_BP_1A: // ............iiii ......pppppddddd -- SVE saturating inc/dec vector by element count - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeUimm4From1_19_to_16(imm); // iiii - dst += emitOutput_Instr(dst, code); - break; + VARSET_TP GCvars(VarSetOps::UninitVal()); - case IF_SVE_BS_1A: // ..............ii iiiiiiiiiiiddddd -- SVE bitwise logical with immediate (unpredicated) - case IF_SVE_BT_1A: // ..............ii iiiiiiiiiiiddddd -- SVE broadcast bitmask immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= (imm << 5); - dst += emitOutput_Instr(dst, code); - break; + // Is this a "fat" call descriptor? + if (id->idIsLargeCall()) + { + instrDescCGCA* idCall = (instrDescCGCA*)id; + gcrefRegs = idCall->idcGcrefRegs; + byrefRegs = idCall->idcByrefRegs; + VarSetOps::Assign(emitComp, GCvars, idCall->idcGCvars); + } + else + { + assert(!id->idIsLargeDsp()); + assert(!id->idIsLargeCns()); - case IF_SVE_BU_2A: // ........xx..gggg ...iiiiiiiiddddd -- SVE copy floating-point immediate (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + gcrefRegs = emitDecodeCallGCregs(id); + byrefRegs = 0; + VarSetOps::AssignNoCopy(emitComp, GCvars, VarSetOps::MakeEmpty(emitComp)); + } - case IF_SVE_BV_2A: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - case IF_SVE_BV_2A_J: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= (id->idHasShift() ? 0x2000 : 0); // h - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + /* We update the GC info before the call as the variables cannot be + used by the call. Killing variables before the call helps with + boundary conditions if the call is CORINFO_HELP_THROW - see bug 50029. + If we ever track aliased variables (which could be used by the + call), we would have to keep them alive past the call. */ - case IF_SVE_BV_2B: // ........xx..gggg ...........ddddd -- SVE copy integer immediate (predicated) - // In emitIns, we set this format's instruction to MOV, as that is the preferred disassembly. - // However, passing (MOV, IF_SVE_BV_2B) to emitInsCodeSve will assert with "encoding_found", - // as FMOV is the only instruction associated with this encoding format. - // Thus, always pass FMOV here, and use MOV elsewhere for simplicity. - code = emitInsCodeSve(INS_sve_fmov, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + emitUpdateLiveGCvars(GCvars, dst); - case IF_SVE_BW_2A: // ........ii.xxxxx ......nnnnnddddd -- SVE broadcast indexed element - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveBroadcastIndex(optGetSveElemsize(id->idInsOpt()), imm); - dst += emitOutput_Instr(dst, code); - break; +#ifdef DEBUG + // Output any delta in GC variable info, corresponding to the before-call GC var updates done above. + if (EMIT_GC_VERBOSE || emitComp->opts.disasmWithGC) + { + emitDispGCVarDelta(); + } +#endif // DEBUG - case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; + // Now output the call instruction and update the 'dst' pointer + // + unsigned outputInstrSize = emitOutput_Instr(dst, code); + dst += outputInstrSize; - case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii - dst += emitOutput_Instr(dst, code); - break; + // All call instructions are 4-byte in size on ARM64 + // + assert(outputInstrSize == callInstrSize); - case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i - dst += emitOutput_Instr(dst, code); - break; + // If the method returns a GC ref, mark INTRET (R0) appropriately. + if (id->idGCref() == GCT_GCREF) + { + gcrefRegs |= RBM_INTRET; + } + else if (id->idGCref() == GCT_BYREF) + { + byrefRegs |= RBM_INTRET; + } - case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii - dst += emitOutput_Instr(dst, code); - break; + // If is a multi-register return method is called, mark INTRET_1 (X1) appropriately + if (id->idIsLargeCall()) + { + instrDescCGCA* idCall = (instrDescCGCA*)id; + if (idCall->idSecondGCref() == GCT_GCREF) + { + gcrefRegs |= RBM_INTRET_1; + } + else if (idCall->idSecondGCref() == GCT_BYREF) + { + byrefRegs |= RBM_INTRET_1; + } + } - case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - dst += emitOutput_Instr(dst, code); - break; + // If the GC register set has changed, report the new set. + if (gcrefRegs != emitThisGCrefRegs) + { + emitUpdateLiveGCregs(GCT_GCREF, gcrefRegs, dst); + } + // If the Byref register set has changed, report the new set. + if (byrefRegs != emitThisByrefRegs) + { + emitUpdateLiveGCregs(GCT_BYREF, byrefRegs, dst); + } - case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii - dst += emitOutput_Instr(dst, code); - break; + // Some helper calls may be marked as not requiring GC info to be recorded. + if ((!id->idIsNoGC())) + { + // On ARM64, as on AMD64, we don't change the stack pointer to push/pop args. + // So we're not really doing a "stack pop" here (note that "args" is 0), but we use this mechanism + // to record the call for GC info purposes. (It might be best to use an alternate call, + // and protect "emitStackPop" under the EMIT_TRACK_STACK_DEPTH preprocessor variable.) + emitStackPop(dst, /*isCall*/ true, callInstrSize, /*args*/ 0); - case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i - dst += emitOutput_Instr(dst, code); - break; + // Do we need to record a call location for GC purposes? + // + if (!emitFullGCinfo) + { + emitRecordGCcall(dst, callInstrSize); + } + } + return callInstrSize; +} - case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Emit a 32-bit Arm64 instruction + */ - case IF_SVE_CC_2A: // ........xx...... ......mmmmmddddd -- SVE insert SIMD&FP scalar register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; +unsigned emitter::emitOutput_Instr(BYTE* dst, code_t code) +{ + assert(sizeof(code_t) == 4); + BYTE* dstRW = dst + writeableOffset; + *((code_t*)dstRW) = code; - case IF_SVE_CD_2A: // ........xx...... ......mmmmmddddd -- SVE insert general register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + return sizeof(code_t); +} - case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeReg_P_19_to_16(id->idReg3()); // MMMM - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** +* + * Append the machine code corresponding to the given instruction descriptor + * to the code block at '*dp'; the base of the code block is 'bp', and 'ig' + * is the instruction group that contains the instruction. Updates '*dp' to + * point past the generated code, and returns the size of the instruction + * descriptor in bytes. + */ - case IF_SVE_CJ_2A: // ........xx...... .......nnnn.dddd -- SVE reverse predicate elements - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; +size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) +{ + BYTE* dst = *dp; + BYTE* odst = dst; + code_t code = 0; + size_t sz = emitGetInstrDescSize(id); // TODO-ARM64-Cleanup: on ARM, this is set in each case. why? + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); + emitAttr size = id->idOpSize(); - case IF_SVE_CK_2A: // ................ .......NNNN.DDDD -- SVE unpack predicate elements - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - dst += emitOutput_Instr(dst, code); - break; +#ifdef DEBUG +#if DUMP_GC_TABLES + bool dspOffs = emitComp->opts.dspGCtbls; +#else + bool dspOffs = !emitComp->opts.disDiffable; +#endif +#endif // DEBUG - case IF_SVE_GQ_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision odd elements - code = emitInsCodeSve(ins, fmt); + assert(REG_NA == (int)REG_NA); - if (ins == INS_sve_fcvtnt && id->idInsOpt() == INS_OPTS_D_TO_S) - { - code |= (1 << 22 | 1 << 17); - } - else if (ins == INS_sve_fcvtlt && id->idInsOpt() == INS_OPTS_S_TO_D) - { - code |= (1 << 22 | 1 << 17); - } + /* What instruction format have we got? */ - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; + switch (fmt) + { + ssize_t imm; + ssize_t index; + ssize_t index2; + unsigned cmode; + unsigned immShift; + emitAttr elemsize; + emitAttr datasize; - // Scalable to general register. - case IF_SVE_CO_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally extract element to general register - case IF_SVE_CS_3A: // ........xx...... ...gggnnnnnddddd -- SVE extract element to general register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_BI_0A: // BI_0A ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 + case IF_BI_0B: // BI_0B ......iiiiiiiiii iiiiiiiiiii..... simm19:00 + case IF_LARGEJMP: + assert(id->idGCref() == GCT_NONE); + assert(id->idIsBound()); + dst = emitOutputLJ(ig, dst, id); + sz = sizeof(instrDescJmp); break; - // Scalable from general register. - case IF_SVE_CQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy general register to vector (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_Rn(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_BI_0C: // BI_0C ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 + code = emitInsCode(ins, fmt); + sz = id->idIsLargeCall() ? sizeof(instrDescCGCA) : sizeof(instrDesc); + dst += emitOutputCall(ig, dst, id, code); + // Always call RecordRelocation so that we wire in a JumpStub when we don't reach + emitRecordRelocation(odst, id->idAddr()->iiaAddr, IMAGE_REL_ARM64_BRANCH26); break; - case IF_SVE_CT_3A: // ................ ...gggnnnnnddddd -- SVE reverse doublewords - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); + case IF_BI_1A: // BI_1A ......iiiiiiiiii iiiiiiiiiiittttt Rt simm19:00 + assert(insOptsNone(id->idInsOpt())); + assert(id->idIsBound()); + + dst = emitOutputLJ(ig, dst, id); + sz = sizeof(instrDescJmp); break; - case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) - case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // VVV - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn/mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_BI_1B: // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 + assert(insOptsNone(id->idInsOpt())); + assert(id->idIsBound()); + + dst = emitOutputLJ(ig, dst, id); + sz = sizeof(instrDescJmp); break; - case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) - { - regNumber reg4 = (ins == INS_sve_mov ? id->idReg1() : id->idReg4()); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_13_to_10(id->idReg2()); // VVVV - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(reg4); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_BR_1A: // BR_1A ................ ......nnnnn..... Rn + assert(insOptsNone(id->idInsOpt())); + assert((ins == INS_ret) || (ins == INS_br)); + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + dst += emitOutput_Instr(dst, code); break; - } - case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors - case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors - case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match - case IF_SVE_HT_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeReg_V_20_to_16(id->idReg4()); // nnnnn - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + case IF_BR_1B: // BR_1B ................ ......nnnnn..... Rn + assert(insOptsNone(id->idInsOpt())); + assert((ins == INS_br_tail) || (ins == INS_blr)); + code = emitInsCode(ins, fmt); - case IF_SVE_CY_3A: // ........xx.iiiii ...gggnnnnn.DDDD -- SVE integer compare with signed immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSimm<20, 16>(imm); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && id->idIsTlsGD()) + { + emitRecordRelocation(odst, (CORINFO_METHOD_HANDLE)id->idAddr()->iiaAddr, + IMAGE_REL_AARCH64_TLSDESC_CALL); + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + } + else + { + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn + } + dst += emitOutputCall(ig, dst, id, code); + sz = id->idIsLargeCall() ? sizeof(instrDescCGCA) : sizeof(instrDesc); break; - case IF_SVE_CY_3B: // ........xx.iiiii ii.gggnnnnn.DDDD -- SVE integer compare with unsigned immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeUimm<20, 14>(imm); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + case IF_LS_1A: // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt PC imm(1MB) + case IF_LARGELDC: + assert(insOptsNone(id->idInsOpt())); + assert(id->idIsBound()); - case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate - case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) - case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product - case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product - case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations - case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long - case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long - case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long - case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); + dst = emitOutputLJ(ig, dst, id); + sz = sizeof(instrDescJmp); break; - case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm - code |= insEncodeReg_V_9_to_5(id->idReg3()); // kkkkk + case IF_LS_2A: // LS_2A .X.......X...... ......nnnnnttttt Rt Rn + assert(insOptsNone(id->idInsOpt())); + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + } + else + { + code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); + if (id->idIsTlsGD()) + { + emitRecordRelocation(odst, (void*)emitGetInsSC(id), IMAGE_REL_AARCH64_TLSDESC_LD64_LO12); + } break; - case IF_SVE_AW_2A: // ........xx.xxiii ......mmmmmddddd -- sve_int_rotate_imm - imm = insGetImmDiff(emitGetInsSC(id), id->idInsOpt()); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm<20, 16>(imm & 0b11111); // xxiii - code |= insEncodeUimm<22, 22>(imm >> 5); // x - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx xx + case IF_LS_2B: // LS_2B .X.......Xiiiiii iiiiiinnnnnttttt Rt Rn imm(0-4095) + assert(insOptsNone(id->idInsOpt())); + imm = emitGetInsSC(id); + assert(isValidUimm<12>(imm)); + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + } + else + { + code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + code |= ((code_t)imm << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate - // increment) + case IF_LS_2C: // LS_2C .X.......X.iiiii iiiiPPnnnnnttttt Rt Rn imm(-256..+255) no/pre/post inc + assert(insOptsNone(id->idInsOpt()) || insOptsIndexed(id->idInsOpt())); + imm = emitGetInsSC(id); + assert((imm >= -256) && (imm <= 255)); // signed 9 bits + imm &= 0x1ff; // force into unsigned 9 bit representation + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) { - ssize_t imm1; - ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm<9, 5>(imm1); // iiiii - code |= insEncodeSimm<20, 16>(imm2); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt } - - case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register - // increment) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm<9, 5>(emitGetInsSC(id)); // iiiii - code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + else + { + code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + code |= insEncodeIndexedOpt(id->idInsOpt()); // PP + code |= ((code_t)imm << 12); // iiiiiiiii + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate - // increment) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSimm<20, 16>(emitGetInsSC(id)); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + case IF_LS_2D: // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn + case IF_LS_2E: // LS_2E .Q.............. ....ssnnnnnttttt Vt Rn + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); - case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm<10, 5>(emitGetInsSC(id)); // iiiiii - code |= insEncodeReg_R_20_to_16(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeVLSElemsize(elemsize); // ss + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vt(id->idReg1()); // ttttt - case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm<10, 5>(emitGetInsSC(id)); // iiiiii dst += emitOutput_Instr(dst, code); break; - case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg3()); // aaaaa - code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm + case IF_LS_2F: // LS_2F .Q.............. xx.Sssnnnnnttttt Vt[] Rn + case IF_LS_2G: // LS_2G .Q.............. xx.Sssnnnnnttttt Vt[] Rn + elemsize = id->idOpSize(); + index = id->idSmallCns(); + code = emitInsCode(ins, fmt); + + code |= insEncodeVLSIndex(elemsize, index); // Q xx S ss + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + dst += emitOutput_Instr(dst, code); break; - case IF_SVE_EG_3A: // ...........iimmm ......nnnnnddddd -- SVE two-way dot product (indexed) - case IF_SVE_EY_3A: // ...........iimmm ......nnnnnddddd -- SVE integer dot product (indexed) - case IF_SVE_EZ_3A: // ...........iimmm ......nnnnnddddd -- SVE mixed sign dot product (indexed) - case IF_SVE_FD_3B: // ...........iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) - case IF_SVE_FF_3B: // ...........iimmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) - case IF_SVE_FI_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) - case IF_SVE_GU_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GX_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) - case IF_SVE_GY_3B: // ...........iimmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) - case IF_SVE_GY_3B_D: // ...........iimmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) - case IF_SVE_FK_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm - code |= insEncodeUimm<20, 19>(emitGetInsSC(id)); // ii + case IF_LS_3A: // LS_3A .X.......X.mmmmm oooS..nnnnnttttt Rt Rn Rm ext(Rm) LSL {} + assert(insOptsLSExtend(id->idInsOpt())); + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + } + else + { + code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + code |= insEncodeExtend(id->idInsOpt()); // ooo + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + if (id->idIsLclVar()) + { + code |= insEncodeReg_Rm(codeGen->rsGetRsvdReg()); // mmmmm + } + else + { + code |= insEncodeReg3Scale(id->idReg3Scaled()); // S + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + } dst += emitOutput_Instr(dst, code); break; - case IF_SVE_FD_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) - case IF_SVE_FF_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) - case IF_SVE_FI_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) - case IF_SVE_GU_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GX_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) - case IF_SVE_FK_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm - code |= insEncodeUimm<20, 19>(imm & 0b11); // ii - code |= insEncodeUimm<22, 22>(imm >> 2); // i + case IF_LS_3B: // LS_3B X............... .aaaaannnnnddddd Rd Ra Rn + assert(insOptsNone(id->idInsOpt())); + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVPLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + code |= insEncodeReg_Va(id->idReg2()); // aaaaa + } + else + { + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + code |= insEncodeReg_Ra(id->idReg2()); // aaaaa + } + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_FE_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) - case IF_SVE_FG_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 integer multiply-add long (indexed) - case IF_SVE_FH_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) - case IF_SVE_FJ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) - case IF_SVE_GY_3A: // ...........iimmm ....i.nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) - case IF_SVE_GZ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE floating-point multiply-add long (indexed) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<11, 11>(imm & 1); // i - code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm - code |= insEncodeUimm<20, 19>(imm >> 1); // ii + case IF_LS_3C: // LS_3C X......PP.iiiiii iaaaaannnnnddddd Rd Ra Rn imm(im7,sh) + assert(insOptsNone(id->idInsOpt()) || insOptsIndexed(id->idInsOpt())); + imm = emitGetInsSC(id); + assert((imm >= -64) && (imm <= 63)); // signed 7 bits + imm &= 0x7f; // force into unsigned 7 bit representation + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVPLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + code |= insEncodeReg_Va(id->idReg2()); // aaaaa + } + else + { + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + code |= insEncodeReg_Ra(id->idReg2()); // aaaaa + } + code |= insEncodePairIndexedOpt(ins, id->idInsOpt()); // PP + code |= ((code_t)imm << 15); // iiiiiiiii + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_FE_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) - case IF_SVE_FG_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply-add long (indexed) - case IF_SVE_FH_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) - case IF_SVE_FJ_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<11, 11>(imm & 1); // i - code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm - code |= insEncodeUimm<20, 19>(imm & 0b10); // i + case IF_LS_3D: // LS_3D .X.......X.mmmmm ......nnnnnttttt Wm Rt Rn + code = emitInsCode(ins, fmt); + // Arm64 store exclusive unpredictable cases + assert(id->idReg1() != id->idReg2()); + assert(id->idReg1() != id->idReg3()); + code |= insEncodeDatasizeLS(code, id->idOpSize()); // X + code |= insEncodeReg_Rm(id->idReg1()); // mmmmm + code |= insEncodeReg_Rt(id->idReg2()); // ttttt + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_EY_3B: // ...........immmm ......nnnnnddddd -- SVE integer dot product (indexed) - case IF_SVE_FD_3C: // ...........immmm ......nnnnnddddd -- SVE2 integer multiply (indexed) - case IF_SVE_FF_3C: // ...........immmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) - case IF_SVE_FI_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) - case IF_SVE_GU_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GX_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply (indexed) - case IF_SVE_FK_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm - - // index is encoded at bit location 20; - // left-shift by one bit so we can reuse insEncodeUimm<20, 19> without modifying bit location 19 - code |= insEncodeUimm<20, 19>(emitGetInsSC(id) << 1); // i + case IF_LS_3E: // LS_3E .X.........mmmmm ......nnnnnttttt Rm Rt Rn ARMv8.1 LSE Atomics + code = emitInsCode(ins, fmt); + code |= insEncodeDatasizeLS(code, id->idOpSize()); // X + code |= insEncodeReg_Rm(id->idReg1()); // mmmmm + code |= insEncodeReg_Rt(id->idReg2()); // ttttt + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CZ_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations - case IF_SVE_DA_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE propagate break from previous partition - { - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - regNumber regm; - switch (ins) + // Some instructions with this encoding return their result in the + // second operand register instead of the first so we special case + // the GC update here and skip the common path down below. + if (emitInsDestIsOp2(ins)) { - case INS_sve_mov: - case INS_sve_movs: - regm = id->idReg3(); - break; - - case INS_sve_not: - case INS_sve_nots: - regm = id->idReg2(); - break; + if (id->idGCref() != GCT_NONE) + { + emitGCregLiveUpd(id->idGCref(), id->idReg2(), dst); + } + else + { + emitGCregDeadUpd(id->idReg2(), dst); + } - default: - regm = id->idReg4(); + goto SKIP_GC_UPDATE; } - code |= insEncodeReg_P_19_to_16(regm); // MMMM - dst += emitOutput_Instr(dst, code); break; - } - case IF_SVE_CZ_4A_A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations - case IF_SVE_CZ_4A_L: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // NNNN - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeReg_P_19_to_16(id->idReg2()); // NNNN - dst += emitOutput_Instr(dst, code); - break; + case IF_LS_3F: // LS_3F .Q.........mmmmm ....ssnnnnnttttt Vt Rn Rm + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); - case IF_SVE_CZ_4A_K: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - code |= insEncodeReg_P_19_to_16(id->idReg1()); // DDDD - dst += emitOutput_Instr(dst, code); - break; + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeVLSElemsize(elemsize); // ss + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vt(id->idReg1()); // ttttt - case IF_SVE_DB_3A: // ................ ..gggg.NNNNMDDDD -- SVE partition break condition - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - code |= insEncodePredQualifier_4(id->idPredicateReg2Merge()); // M dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DB_3B: // ................ ..gggg.NNNN.DDDD -- SVE partition break condition - case IF_SVE_DC_3A: // ................ ..gggg.NNNN.MMMM -- SVE propagate break to next partition - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - dst += emitOutput_Instr(dst, code); - break; + case IF_LS_3G: // LS_3G .Q.........mmmmm ...Sssnnnnnttttt Vt[] Rn Rm + elemsize = id->idOpSize(); + index = id->idSmallCns(); + code = emitInsCode(ins, fmt); + + code |= insEncodeVLSIndex(elemsize, index); // Q xx S ss + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vt(id->idReg1()); // ttttt - case IF_SVE_DD_2A: // ................ .......gggg.DDDD -- SVE predicate first active - case IF_SVE_DG_2A: // ................ .......gggg.DDDD -- SVE predicate read from FFR (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // gggg dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DE_1A: // ........xx...... ......ppppp.DDDD -- SVE predicate initialize - code = emitInsCodeSve(ins, fmt); - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DI_1A: // DI_1A X.......shiiiiii iiiiiinnnnn..... Rn imm(i12,sh) + assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); + imm = emitGetInsSC(id); + assert(isValidUimm<12>(imm)); + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeShiftImm12(id->idInsOpt()); // sh + code |= ((code_t)imm << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DF_2A: // ........xx...... .......VVVV.DDDD -- SVE predicate next active - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // VVVV - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DI_1B: // DI_1B X........hwiiiii iiiiiiiiiiiddddd Rd imm(i16,hw) + imm = emitGetInsSC(id); + assert(isValidImmHWVal(imm, id->idOpSize())); + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= ((code_t)imm << 5); // hwiiiii iiiiiiiiiii + code |= insEncodeReg_Rd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DH_1A: // ................ ............DDDD -- SVE predicate read from FFR (unpredicated) - case IF_SVE_DJ_1A: // ................ ............DDDD -- SVE predicate zero - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + case IF_DI_1C: // DI_1C X........Nrrrrrr ssssssnnnnn..... Rn imm(N,r,s) + imm = emitGetInsSC(id); + assert(isValidImmNRS(imm, id->idOpSize())); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 10); // Nrrrrrrssssss + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DI_2A: // ................ ..gggg.NNNN..... -- SVE predicate test - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_13_to_10(id->idReg1()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + case IF_DI_1D: // DI_1D X........Nrrrrrr ssssss.....ddddd Rd imm(N,r,s) + imm = emitGetInsSC(id); + assert(isValidImmNRS(imm, id->idOpSize())); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 10); // Nrrrrrrssssss + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DK_3A: // ........xx...... ..gggg.NNNNddddd -- SVE predicate count - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_DI_1E: // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 + case IF_LARGEADR: + assert(insOptsNone(id->idInsOpt())); + if (id->idIsReloc()) + { + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + dst += emitOutput_Instr(dst, code); + emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21 + : IMAGE_REL_ARM64_PAGEBASE_REL21); + } + else + { + // Local jmp/load case which does not need a relocation. + assert(id->idIsBound()); + dst = emitOutputLJ(ig, dst, id); + } + sz = sizeof(instrDescJmp); break; - case IF_SVE_GA_2A: // ............iiii ......nnnn.ddddd -- SME2 multi-vec shift narrow + case IF_DI_1F: // DI_1F X..........iiiii cccc..nnnnn.nzcv Rn imm5 nzcv cond imm = emitGetInsSC(id); - assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); - assert(emitInsIsVectorRightShift(id->idIns())); - assert(isValidVectorShiftAmount(imm, EA_4BYTE, /* rightShift */ true)); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeVectorShift(EA_4BYTE, true /* right-shift */, imm); // iiii - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_6_Times_Two(id->idReg2()); // nnnn - dst += emitOutput_Instr(dst, code); + assert(isValidImmCondFlagsImm5(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= ((code_t)cfi.imm5 << 16); // iiiii + code |= insEncodeFlags(cfi.flags); // nzcv + code |= insEncodeCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } break; - case IF_SVE_DL_2A: // ........xx...... .....l.NNNNddddd -- SVE predicate count (predicate-as-counter) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeVectorLengthSpecifier(id); // l - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DI_2A: // DI_2A X.......shiiiiii iiiiiinnnnnddddd Rd Rn imm(i12,sh) + assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); + imm = emitGetInsSC(id); + assert(isValidUimm<12>(imm)); + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeShiftImm12(id->idInsOpt()); // sh + code |= ((code_t)imm << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); - break; - case IF_SVE_DM_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec register by predicate count - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + if (id->idIsReloc()) + { + assert(sz == sizeof(instrDesc)); + assert(id->idAddr()->iiaAddr != nullptr); + emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADD_LO12 + : IMAGE_REL_ARM64_PAGEOFFSET_12A); + } break; - case IF_SVE_DN_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec vector by predicate count - case IF_SVE_DP_2A: // ........xx...... .......MMMMddddd -- SVE saturating inc/dec vector by predicate count - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DI_2B: // DI_2B X.........Xnnnnn ssssssnnnnnddddd Rd Rn imm(0-63) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // Reg2 also in mmmmm + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DO_2A: // ........xx...... .....X.MMMMddddd -- SVE saturating inc/dec register by predicate count - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM - code |= insEncodeVLSElemsize(id->idOpSize()); // X - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DI_2C: // DI_2C X........Nrrrrrr ssssssnnnnnddddd Rd Rn imm(N,r,s) + imm = emitGetInsSC(id); + assert(isValidImmNRS(imm, id->idOpSize())); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 10); // Nrrrrrrssssss + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DQ_0A: // ................ ................ -- SVE FFR initialise - code = emitInsCodeSve(ins, fmt); - dst += emitOutput_Instr(dst, code); - break; + case IF_DI_2D: // DI_2D X........Nrrrrrr ssssssnnnnnddddd Rd Rn imr, imms (N,r,s) + if (ins == INS_asr || ins == INS_lsl || ins == INS_lsr) + { + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); - case IF_SVE_DR_1A: // ................ .......NNNN..... -- SVE FFR write from predicate - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_8_to_5(id->idReg1()); // NNNN - dst += emitOutput_Instr(dst, code); - break; + // Shift immediates are aliases of the SBFM/UBFM instructions + // that actually take 2 registers and 2 constants, + // Since we stored the shift immediate value + // we need to calculate the N,R and S values here. - case IF_SVE_DS_2A: // .........x.mmmmm ......nnnnn..... -- SVE conditionally terminate scalars - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_9_to_5(id->idReg1()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm - code |= insEncodeSveElemsize_R_22(id->idOpSize()); // x - dst += emitOutput_Instr(dst, code); - break; + bitMaskImm bmi; + bmi.immNRS = 0; - case IF_SVE_FZ_2A: // ................ ......nnnn.ddddd -- SME2 multi-vec extract narrow - case IF_SVE_HG_2A: // ................ ......nnnn.ddddd -- SVE2 FP8 downconverts - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_6(id->idReg2()); // nnnn - dst += emitOutput_Instr(dst, code); - break; + bmi.immN = (size == EA_8BYTE) ? 1 : 0; + bmi.immR = imm; + bmi.immS = (size == EA_8BYTE) ? 0x3f : 0x1f; - case IF_SVE_GD_2A: // .........x.xx... ......nnnnnddddd -- SVE2 saturating extract narrow - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - // Bit 23 should not be set by below call - assert(insOptsScalableWide(id->idInsOpt())); - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx - // x + // immR and immS are now set correctly for INS_asr and INS_lsr + // but for INS_lsl we have to adjust the values for immR and immS + // + if (ins == INS_lsl) + { + bmi.immR = -imm & bmi.immS; + bmi.immS = bmi.immS - imm; + } + + // setup imm with the proper 13 bit value N:R:S + // + imm = bmi.immNRS; + } + else + { + // The other instructions have already have encoded N,R and S values + imm = emitGetInsSC(id); + } + assert(isValidImmNRS(imm, id->idOpSize())); + + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 10); // Nrrrrrrssssss + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<20, 16>(emitGetInsSC(id)); // iii - // Bit 23 should not be set by below call - assert(insOptsScalableWide(id->idInsOpt())); - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx - // x - dst += emitOutput_Instr(dst, code); + case IF_DR_1D: // DR_1D X............... cccc.......ddddd Rd cond + imm = emitGetInsSC(id); + assert(isValidImmCond(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeInvertedCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } break; - case IF_SVE_GB_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift right narrow - // Bit 23 should not be set by call to insEncodeSveElemsize_tszh_23_tszl_20_to_19, - // nor should we pass INS_OPTS_SCALABLE_D to insGetImmDiff. - assert(insOptsScalableWide(id->idInsOpt())); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<20, 16>(insGetImmDiff(emitGetInsSC(id), id->idInsOpt())); // iii - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx - // x + case IF_DR_2A: // DR_2A X..........mmmmm ......nnnnn..... Rn Rm + assert(insOptsNone(id->idInsOpt())); + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_FV_2A: // ........xx...... .....rmmmmmddddd -- SVE2 complex integer add - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm<10, 10>(emitGetInsSC(id)); // r - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DR_2B: // DR_2B X.......sh.mmmmm ssssssnnnnn..... Rn Rm {LSL,LSR,ASR,ROR} imm(0-63) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeShiftType(id->idInsOpt()); // sh + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_FY_3A: // .........x.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long with carry - { - // Size encoding: 1 if INS_OPTS_SCALABLE_D, 0 if INS_OPTS_SCALABLE_S - const ssize_t sizeEncoding = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? 1 : 0; - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm<22, 22>(sizeEncoding); // x + case IF_DR_2C: // DR_2C X..........mmmmm ooosssnnnnn..... Rn Rm ext(Rm) LSL imm(0-4) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert((imm >= 0) && (imm <= 4)); // imm [0..4] + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeExtend(id->idInsOpt()); // ooo + code |= insEncodeExtendScale(imm); // sss + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - } - case IF_SVE_GK_2A: // ................ ......mmmmmddddd -- SVE2 crypto destructive binary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); + case IF_DR_2D: // DR_2D X..........nnnnn cccc..nnnnnddddd Rd Rn cond + imm = emitGetInsSC(id); + assert(isValidImmCond(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm + code |= insEncodeInvertedCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } break; - case IF_SVE_GL_1A: // ................ ...........ddddd -- SVE2 crypto unary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + case IF_DR_2E: // DR_2E X..........mmmmm ...........ddddd Rd Rm + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DT_3A: // ........xx.mmmmm ...X..nnnnn.DDDD -- SVE integer compare scalar count and limit - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= (id->idOpSize() == EA_8BYTE) ? (1 << 12) : 0; // X - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DR_2F: // DR_2F X.......sh.mmmmm ssssss.....ddddd Rd Rm {LSL,LSR,ASR} imm(0-63) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeShiftType(id->idInsOpt()); // sh + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DW_2A: // ........xx...... ......iiNNN.DDDD -- SVE extract mask predicate from predicate-as-counter - case IF_SVE_DW_2B: // ........xx...... .......iNNN.DDDD -- SVE extract mask predicate from predicate-as-counter - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_7_to_5(id->idReg2()); // NNN - code |= insEncodeUimm<9, 8>(emitGetInsSC(id)); // ii (or i) - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DR_2G: // DR_2G X............... .....xnnnnnddddd Rd Rn + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + if (ins == INS_rev) + { + if (size == EA_8BYTE) + { + code |= 0x00000400; // x - bit at location 10 + } + } + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DX_3A: // ........xx.mmmmm ......nnnnn.DDD. -- SVE integer compare scalar count and limit (predicate - // pair) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_1(id->idReg1()); // DDD - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DR_2H: // DR_2H X........X...... ......nnnnnddddd Rd Rn + code = emitInsCode(ins, fmt); + code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DY_3A: // ........xx.mmmmm ..l...nnnnn..DDD -- SVE integer compare scalar count and limit - // (predicate-as-counter) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeVectorLengthSpecifier(id); // l - code |= insEncodeReg_P_2_to_0(id->idReg1()); // DDD - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_DR_2I: // DR_2I X..........mmmmm cccc..nnnnn.nzcv Rn Rm nzcv cond + imm = emitGetInsSC(id); + assert(isValidImmCondFlags(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm + code |= insEncodeFlags(cfi.flags); // nzcv + code |= insEncodeCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } break; - case IF_SVE_DZ_1A: // ........xx...... .............DDD -- sve_int_pn_ptrue - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_2_to_0(id->idReg1()); // DDD - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DR_3A: // DR_3A X..........mmmmm ......nnnnnmmmmm Rd Rn Rm + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + if (id->idIsLclVar()) + { + code |= insEncodeReg_Rm(codeGen->rsGetRsvdReg()); // mmmmm + } + else + { + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + } dst += emitOutput_Instr(dst, code); break; - case IF_SVE_EA_1A: // ........xx...... ...iiiiiiiiddddd -- SVE broadcast floating-point immediate (unpredicated) - case IF_SVE_ED_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer min/max immediate (unpredicated) - case IF_SVE_EE_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer multiply immediate (unpredicated) - { + case IF_DR_3B: // DR_3B X.......sh.mmmmm ssssssnnnnnddddd Rd Rn Rm {LSL,LSR,ASR} imm(0-63) + code = emitInsCode(ins, fmt); imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - } - - case IF_SVE_FA_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) - case IF_SVE_FB_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add (indexed) - case IF_SVE_FC_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex saturating multiply-add (indexed) - { - const ssize_t imm = emitGetInsSC(id); - const ssize_t rot = (imm & 0b11); - const ssize_t index = (imm >> 2); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<11, 10>(rot); // rr - code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm - code |= insEncodeUimm<20, 19>(index); // ii - dst += emitOutput_Instr(dst, code); - break; - } - - case IF_SVE_EJ_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer dot product - case IF_SVE_EK_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // rr - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_FA_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) - case IF_SVE_FB_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add (indexed) - case IF_SVE_FC_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex saturating multiply-add (indexed) - case IF_SVE_GV_3A: // ...........immmm ....rrnnnnnddddd -- SVE floating-point complex multiply-add (indexed) - { - const ssize_t imm = emitGetInsSC(id); - const ssize_t rot = (imm & 0b11); - const ssize_t index = (imm >> 2); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm - code |= insEncodeUimm<11, 10>(rot); // rr - - // index is encoded at bit location 20; - // left-shift by one bit so we can reuse insEncodeUimm<20, 19> without modifying bit location 19 - code |= insEncodeUimm<20, 19>(index << 1); // i + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeShiftType(id->idInsOpt()); // sh + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss dst += emitOutput_Instr(dst, code); break; - } - case IF_SVE_EB_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE broadcast integer immediate (unpredicated) - case IF_SVE_EC_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE integer add/subtract immediate (unpredicated) + case IF_DR_3C: // DR_3C X..........mmmmm ooosssnnnnnddddd Rd Rn Rm ext(Rm) LSL imm(0-4) + code = emitInsCode(ins, fmt); imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= (id->idHasShift() ? 0x2000 : 0); // h + assert((imm >= 0) && (imm <= 4)); // imm [0..4] + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeExtend(id->idInsOpt()); // ooo + code |= insEncodeExtendScale(imm); // sss + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_EB_1B: // ........xx...... ...........ddddd -- SVE broadcast integer immediate (unpredicated) - // ins is MOV for this encoding, as it is the preferred disassembly, so pass FMOV to emitInsCodeSve - code = emitInsCodeSve(INS_sve_fmov, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_DR_3D: // DR_3D X..........mmmmm cccc..nnnnnddddd Rd Rn Rm cond + imm = emitGetInsSC(id); + assert(isValidImmCond(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } break; - case IF_SVE_DU_3A: // ........xx.mmmmm ......nnnnn.DDDD -- SVE pointer conflict compare - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DR_3E: // DR_3E X........X.mmmmm ssssssnnnnnddddd Rd Rn Rm imm(0-63) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DV_4A: // ........ix.xxxvv ..NNNN.MMMM.DDDD -- SVE broadcast predicate element - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // NNNN - code |= insEncodeReg_P_8_to_5(id->idReg3()); // MMMM - code |= insEncodeReg_R_17_to_16(id->idReg4()); // vv - code |= insEncodeSveElemsize_tszh_tszl_and_imm(id->idInsOpt(), emitGetInsSC(id)); // ix xx + case IF_DR_4A: // DR_4A X..........mmmmm .aaaaannnnnmmmmm Rd Rn Rm Ra + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeReg_Ra(id->idReg4()); // aaaaa dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HO_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + case IF_DV_1A: // DV_1A .........X.iiiii iii........ddddd Vd imm8 (fmov - immediate scalar) + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= ((code_t)imm << 13); // iiiii iii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HO_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - switch (id->idInsOpt()) - { - case INS_OPTS_H_TO_S: - code |= (1 << 16); - break; - case INS_OPTS_H_TO_D: - code |= (1 << 22) | (1 << 16); - break; - case INS_OPTS_S_TO_H: + case IF_DV_1B: // DV_1B .QX..........iii cmod..iiiiiddddd Vd imm8 (immediate vector) + imm = emitGetInsSC(id) & 0x0ff; + immShift = (emitGetInsSC(id) & 0x700) >> 8; + elemsize = optGetElemsize(id->idInsOpt()); + cmode = 0; + switch (elemsize) + { // cmode + case EA_1BYTE: + cmode = 0xE; // 1110 break; - case INS_OPTS_S_TO_D: - code |= (1 << 22) | (3 << 16); + case EA_2BYTE: + cmode = 0x8; + cmode |= (immShift << 1); // 10x0 break; - case INS_OPTS_D_TO_H: - code |= (1 << 22); + case EA_4BYTE: + if (immShift < 4) + { + cmode = 0x0; + cmode |= (immShift << 1); // 0xx0 + } + else // MSL + { + cmode = 0xC; + if (immShift & 2) + cmode |= 1; // 110x + } break; - case INS_OPTS_D_TO_S: - code |= (1 << 22) | (1 << 17); + case EA_8BYTE: + cmode = 0xE; // 1110 break; default: unreached(); + break; + } + + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + if ((ins == INS_fmov) || (ins == INS_movi)) + { + if (elemsize == EA_8BYTE) + { + code |= 0x20000000; // X + } + } + if (ins != INS_fmov) + { + assert((cmode >= 0) && (cmode <= 0xF)); + code |= (cmode << 12); // cmod } + code |= (((code_t)imm >> 5) << 16); // iii + code |= (((code_t)imm & 0x1f) << 5); // iiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HO_3C: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + case IF_DV_1C: // DV_1C .........X...... ......nnnnn..... Vn #0.0 (fcmp - with zero) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vn(id->idReg1()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HP_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert to integer - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - - switch (id->idInsOpt()) + case IF_DV_2A: // DV_2A .Q.......X...... ......nnnnnddddd Vd Vn (fabs, fcvt - vector) + case IF_DV_2R: // DV_2R .Q.......X...... ......nnnnnddddd Sd Vn (fmaxnmv, fmaxv, fminnmv, fminv) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + if ((ins == INS_fcvtl) || (ins == INS_fcvtl2) || (ins == INS_fcvtn) || (ins == INS_fcvtn2)) { - case INS_OPTS_SCALABLE_H: - code |= (1 << 22) | (1 << 17); - break; - case INS_OPTS_H_TO_S: - code |= (1 << 22) | (1 << 18); - break; - case INS_OPTS_H_TO_D: - code |= (1 << 22) | (3 << 17); - break; - case INS_OPTS_SCALABLE_S: - code |= (1 << 23) | (1 << 18); - break; - case INS_OPTS_S_TO_D: - code |= (3 << 22) | (1 << 18); - break; - case INS_OPTS_D_TO_S: - code |= (3 << 22); - break; - case INS_OPTS_SCALABLE_D: - code |= (3 << 22) | (3 << 17); - break; - default: - unreached(); - break; + // fcvtl{2} and fcvtn{2} encode the element size as + // esize = 16 << UInt(sz) + if (elemsize == EA_4BYTE) + { + code |= 0x00400000; // X + } + else + { + assert(elemsize == EA_2BYTE); + } } - + else + { + code |= insEncodeFloatElemsize(elemsize); // X + } + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HS_3A: // ................ ...gggnnnnnddddd -- SVE integer convert to floating-point - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - - switch (id->idInsOpt()) + case IF_DV_2B: // DV_2B .Q.........iiiii ......nnnnnddddd Rd Vn[] (umov/smov - to general) + elemsize = id->idOpSize(); + index = emitGetInsSC(id); + datasize = (elemsize == EA_8BYTE) ? EA_16BYTE : EA_8BYTE; + if (ins == INS_smov) { - case INS_OPTS_SCALABLE_H: - code |= (1 << 22) | (1 << 17); - break; - case INS_OPTS_S_TO_H: - code |= (1 << 22) | (1 << 18); - break; - case INS_OPTS_SCALABLE_S: - code |= (1 << 23) | (1 << 18); - break; - case INS_OPTS_S_TO_D: - code |= (1 << 23) | (1 << 22); - break; - case INS_OPTS_D_TO_H: - code |= (1 << 22) | (3 << 17); - break; - case INS_OPTS_D_TO_S: - code |= (3 << 22) | (1 << 18); - break; - case INS_OPTS_SCALABLE_D: - code |= (3 << 22) | (3 << 17); - break; - default: - unreached(); - break; + datasize = EA_16BYTE; } - + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(datasize); // Q + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus - // immediate) - case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus - // immediate) - case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus - // immediate) - case IF_SVE_IJ_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IJ_3A_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IJ_3A_E: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IJ_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IJ_3A_G: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) - case IF_SVE_IL_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus - // immediate) - case IF_SVE_IL_3A_B: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus - // immediate) - case IF_SVE_IL_3A_C: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus - // immediate) - case IF_SVE_IM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus - // immediate) - case IF_SVE_IO_3A: // ............iiii ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus - // immediate) - case IF_SVE_IQ_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus - // immediate) - case IF_SVE_IS_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (scalar plus immediate) - case IF_SVE_JE_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus - // immediate) - case IF_SVE_JM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus - // immediate) - case IF_SVE_JN_3C: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) - case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) - case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - - switch (ins) + case IF_DV_2C: // DV_2C .Q.........iiiii ......nnnnnddddd Vd Rn (dup/ins - vector from general) + if (ins == INS_dup) { - case INS_sve_ld2b: - case INS_sve_ld2h: - case INS_sve_ld2w: - case INS_sve_ld2d: - case INS_sve_ld2q: - case INS_sve_st2b: - case INS_sve_st2h: - case INS_sve_st2w: - case INS_sve_st2d: - case INS_sve_st2q: - code |= insEncodeSimm4_MultipleOf2_19_to_16(imm); // iiii - break; - - case INS_sve_ld3b: - case INS_sve_ld3h: - case INS_sve_ld3w: - case INS_sve_ld3d: - case INS_sve_ld3q: - case INS_sve_st3b: - case INS_sve_st3h: - case INS_sve_st3w: - case INS_sve_st3d: - case INS_sve_st3q: - code |= insEncodeSimm4_MultipleOf3_19_to_16(imm); // iiii - break; - - case INS_sve_ld4b: - case INS_sve_ld4h: - case INS_sve_ld4w: - case INS_sve_ld4d: - case INS_sve_ld4q: - case INS_sve_st4b: - case INS_sve_st4h: - case INS_sve_st4w: - case INS_sve_st4d: - case INS_sve_st4q: - code |= insEncodeSimm4_MultipleOf4_19_to_16(imm); // iiii - break; - - case INS_sve_ld1rqb: - case INS_sve_ld1rqd: - case INS_sve_ld1rqh: - case INS_sve_ld1rqw: - code |= insEncodeSimm4_MultipleOf16_19_to_16(imm); // iiii - break; - - case INS_sve_ld1rob: - case INS_sve_ld1rod: - case INS_sve_ld1roh: - case INS_sve_ld1row: - code |= insEncodeSimm4_MultipleOf32_19_to_16(imm); // iiii - break; - - default: - code |= insEncodeSimm<19, 16>(imm); // iiii - break; + datasize = id->idOpSize(); + elemsize = optGetElemsize(id->idInsOpt()); + index = 0; } - - if (canEncodeSveElemsize_dtype(ins)) + else // INS_ins { - if (ins == INS_sve_ld1w) - { - code = insEncodeSveElemsize_dtype_ld1w(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); - } - else - { - code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); - } + datasize = EA_16BYTE; + elemsize = id->idOpSize(); + index = emitGetInsSC(id); } + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(datasize); // Q + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + case IF_DV_2D: // DV_2D .Q.........iiiii ......nnnnnddddd Vd Vn[] (dup - vector) + index = emitGetInsSC(id); + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JD_4A: // .........xxmmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm - code |= insEncodeSveElemsize_22_to_21(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2E: // DV_2E ...........iiiii ......nnnnnddddd Vd Vn[] (dup - scalar) + index = emitGetInsSC(id); + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JD_4B: // ..........xmmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm - code |= insEncodeSveElemsize_sz_21(optGetSveElemsize(id->idInsOpt())); // x + case IF_DV_2F: // DV_2F ...........iiiii .jjjj.nnnnnddddd Vd[] Vn[] (ins - element) + elemsize = id->idOpSize(); + imm = emitGetInsSC(id); + index = (imm >> 4) & 0xf; + index2 = imm & 0xf; + code = emitInsCode(ins, fmt); + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeVectorIndex2(elemsize, index2); // jjjj + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JJ_4A: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JJ_4A_B: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // // offsets) - case IF_SVE_JJ_4A_C: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JJ_4A_D: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JK_4A: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit unscaled - // offsets) - case IF_SVE_JK_4A_B: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit - // unscaled offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2G: // DV_2G .........X...... ......nnnnnddddd Vd Vn (fmov, fcvtXX - register) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - code |= (1 << 14); // h - break; + case IF_DV_2H: // DV_2H X........X...... ......nnnnnddddd Rd Vn (fmov - to general) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // X X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; - default: - break; - } + case IF_DV_2I: // DV_2I X........X...... ......nnnnnddddd Vd Rn (fmov - from general) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // X X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + case IF_DV_2J: // DV_2J ........SS.....D D.....nnnnnddddd Vd Vn (fcvt) + code = emitInsCode(ins, fmt); + code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // SS DD + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JN_3A: // .........xx.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSimm<19, 16>(imm); // iiii - code |= insEncodeSveElemsize_22_to_21(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2K: // DV_2K .........X.mmmmm ......nnnnn..... Vn Vm (fcmp) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vn(id->idReg1()); // nnnnn + code |= insEncodeReg_Vm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JN_3B: // ..........x.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSimm<19, 16>(imm); // iiii - code |= insEncodeSveElemsize_sz_21(optGetSveElemsize(id->idInsOpt())); // x + case IF_DV_2L: // DV_2L ........XX...... ......nnnnnddddd Vd Vn (abs, neg - scalar) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HW_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_HW_4A_A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_HW_4A_B: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_HW_4A_C: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_IU_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_IU_4A_A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_IU_4A_C: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm - - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - code |= (1 << 22); // h - break; - - default: - break; - } - + case IF_DV_2M: // DV_2M .Q......XX...... ......nnnnnddddd Vd Vn (abs, neg - vector) + case IF_DV_2T: // DV_2T .Q......XX...... ......nnnnnddddd Sd Vn (addv, saddlv, smaxv, sminv, uaddlv, + // umaxv, uminv) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HW_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_HW_4B_D: // ...........mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2N: // DV_2N .........iiiiiii ......nnnnnddddd Vd Vn imm (shift - scalar) + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorShift(elemsize, emitInsIsVectorRightShift(ins), imm); // iiiiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IF_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit gather non-temporal load (vector plus - // scalar) - case IF_SVE_IF_4A_A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit gather non-temporal load (vector plus - // scalar) - case IF_SVE_IW_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 128-bit gather load (vector plus scalar) - case IF_SVE_IX_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 64-bit gather non-temporal load (vector plus - // scalar) - case IF_SVE_IY_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 128-bit scatter store (vector plus scalar) - case IF_SVE_IZ_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit scatter non-temporal store (vector plus - // scalar) - case IF_SVE_IZ_4A_A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit scatter non-temporal store (vector plus - // scalar) - case IF_SVE_JA_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 64-bit scatter non-temporal store (vector plus - // scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2O: // DV_2O .Q.......iiiiiii ......nnnnnddddd Vd Vn imm (shift - vector) + imm = emitGetInsSC(id); + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeVectorShift(elemsize, emitInsIsVectorRightShift(ins), imm); // iiiiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IG_4A_D: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus - // scalar) - case IF_SVE_IG_4A_E: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus - // scalar) - case IF_SVE_IG_4A_F: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus - // scalar) - case IF_SVE_IG_4A_G: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus - // scalar) - case IF_SVE_II_4A_H: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) - case IF_SVE_IK_4A_F: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - case IF_SVE_IK_4A_G: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - case IF_SVE_IK_4A_H: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - case IF_SVE_IK_4A_I: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm - - if (canEncodeSveElemsize_dtype(ins)) - { - if (ins == INS_sve_ld1w) - { - code = insEncodeSveElemsize_dtype_ld1w(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); - } - else - { - code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); - } - } - + case IF_DV_2P: // DV_2P ............... ......nnnnnddddd Vd Vn (aes*, sha1su1) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IG_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus scalar) - case IF_SVE_II_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) - case IF_SVE_II_4A_B: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) - case IF_SVE_IK_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - case IF_SVE_IN_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus scalar) - case IF_SVE_IP_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus scalar) - case IF_SVE_IR_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus - // scalar) - case IF_SVE_IT_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load multiple structures (scalar plus scalar) - case IF_SVE_JB_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus - // scalar) - case IF_SVE_JC_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE store multiple structures (scalar plus scalar) - case IF_SVE_JD_4C: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) - case IF_SVE_JD_4C_A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) - case IF_SVE_JF_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus - // scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2Q: // DV_2Q .........X...... ......nnnnnddddd Vd Vn (faddp, fmaxnmp, fmaxp, fminnmp, + // fminp - scalar) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IU_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_IU_4B_B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_IU_4B_D: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_JJ_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JJ_4B_C: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JJ_4B_E: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JK_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit unscaled - // offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2S: // DV_2S ........XX...... ......nnnnnddddd Sd Vn (addp - scalar) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GP_3A: // ........xx.....r ...gggmmmmmddddd -- SVE floating-point complex add (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeSveImm90_or_270_rot(imm); // r - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2U: // DV_2U ................ ......nnnnnddddd Sd Sn (sha1h) + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GT_4A: // ........xx.mmmmm .rrgggnnnnnddddd -- SVE floating-point complex multiply-add (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm - code |= insEncodeSveImm0_to_270_rot(imm); // rr - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_3A: // DV_3A .Q......XX.mmmmm ......nnnnnddddd Vd Vn Vm (vector) + code = emitInsCode(ins, fmt); + elemsize = optGetElemsize(id->idInsOpt()); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HI_3A: // ........xx...... ...gggnnnnn.DDDD -- SVE floating-point compare with zero - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_3AI: // DV_3AI .Q......XXLMmmmm ....H.nnnnnddddd Vd Vn Vm[] (vector) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + elemsize = optGetElemsize(id->idInsOpt()); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeVectorIndexLMH(elemsize, imm); // LM H + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HM_2A: // ........xx...... ...ggg....iddddd -- SVE floating-point arithmetic with immediate - // (predicated) - { - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeSveSmallFloatImm(imm); // i - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - } - break; - - case IF_SVE_HN_2A: // ........xx...iii ......mmmmmddddd -- SVE floating-point trig multiply-add coefficient - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm<18, 16>(imm); // iii - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_3B: // DV_3B .Q.......X.mmmmm ......nnnnnddddd Vd Vn Vm (vector) + code = emitInsCode(ins, fmt); + elemsize = optGetElemsize(id->idInsOpt()); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HP_3A: // .............xx. ...gggnnnnnddddd -- SVE floating-point convert to integer - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSveElemsize_18_to_17(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_3BI: // DV_3BI .Q.......XLmmmmm ....H.nnnnnddddd Vd Vn Vm[] (vector by element) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + elemsize = optGetElemsize(id->idInsOpt()); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeFloatIndex(elemsize, imm); // L H + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HU_4B: // ...........mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + case IF_DV_3C: // DV_3C .Q.........mmmmm ......nnnnnddddd Vd Vn Vm (vector) + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HV_4A: // ........xx.aaaaa ...gggmmmmmddddd -- SVE floating-point multiply-accumulate writing - // multiplicand - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeReg_V_20_to_16(id->idReg4()); // aaaaa - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_3D: // DV_3D .........X.mmmmm ......nnnnnddddd Vd Vn Vm (scalar) + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(id->idOpSize()); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_ID_2A: // ..........iiiiii ...iiinnnnn.TTTT -- SVE load predicate register - case IF_SVE_JG_2A: // ..........iiiiii ...iiinnnnn.TTTT -- SVE store predicate register - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // TTTT - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSimm9h9l_21_to_16_and_12_to_10(imm); // iii - // iiiiii + case IF_DV_3DI: // DV_3DI .........XLmmmmm ....H.nnnnnddddd Vd Vn Vm[] (scalar by element) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeFloatIndex(elemsize, imm); // L H + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IE_2A: // ..........iiiiii ...iiinnnnnttttt -- SVE load vector register - case IF_SVE_JH_2A: // ..........iiiiii ...iiinnnnnttttt -- SVE store vector register - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSimm9h9l_21_to_16_and_12_to_10(imm); // iii - // iiiiii + case IF_DV_3E: // DV_3E ........XX.mmmmm ......nnnnnddddd Vd Vn Vm (scalar) + code = emitInsCode(ins, fmt); + elemsize = id->idOpSize(); + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm<23, 22>(imm); // ii + case IF_DV_3EI: // DV_3EI ........XXLMmmmm ....H.nnnnnddddd Vd Vn Vm[] (scalar by element) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeVectorIndexLMH(elemsize, imm); // LM H + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm3h3l_23_to_22_and_12(imm); // ii - // i + case IF_DV_3F: // DV_3F ...........mmmmm ......nnnnnddddd Vd Vn Vm (vector) - source dest regs overlap + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size + case IF_DV_3G: // DV_3G .Q.........mmmmm .iiii.nnnnnddddd Vd Vn Vm imm (vector) imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm1_23(imm); // i + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm + code |= ((code_t)imm << 11); // iiii + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit - // scaled offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo - - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - code |= (1 << 22); // h - break; - - default: - break; - } - + case IF_DV_4A: // DV_4A .........X.mmmmm .aaaaannnnnddddd Vd Va Vn Vm (scalar) + code = emitInsCode(ins, fmt); + elemsize = id->idOpSize(); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm + code |= insEncodeReg_Va(id->idReg4()); // aaaaa dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HY_3B: // ...........mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo - dst += emitOutput_Instr(dst, code); - break; + case IF_SN_0A: // SN_0A ................ ................ + { + bool skipIns = false; +#if FEATURE_LOOP_ALIGN + if (id->idIns() == INS_align) + { + // IG can be marked as not needing alignment after emitting align instruction. + // Alternatively, there are fewer align instructions needed than emitted. + // If that is the case, skip outputting alignment. + if (!ig->endsWithAlignInstr() || id->idIsEmptyAlign()) + { + skipIns = true; + } - case IF_SVE_IB_3A: // ...........mmmmm ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo - dst += emitOutput_Instr(dst, code); - break; +#ifdef DEBUG + if (!ig->endsWithAlignInstr()) + { + // Validate if the state is correctly updated + assert(id->idIsEmptyAlign()); + } +#endif + sz = sizeof(instrDescAlign); + ins = INS_nop; - case IF_SVE_HZ_2A_B: // ...........iiiii ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= id->idSvePrfop(); // oooo +#ifdef DEBUG + // Under STRESS_EMITTER, if this is the 'align' before the 'jmp' instruction, + // then add "bkpt" instruction. + instrDescAlign* alignInstr = (instrDescAlign*)id; - if (id->idInsOpt() == INS_OPTS_SCALABLE_D) - { - code |= (1 << 30); // set bit '30' to make it a double-word + if (emitComp->compStressCompile(Compiler::STRESS_EMITTER, 50) && alignInstr->isPlacedAfterJmp && + !skipIns) + { + // There is no good way to squeeze in "bkpt" as well as display it + // in the disassembly because there is no corresponding instrDesc for + // it. As such, leave it as is, the "0xD43E0000" bytecode will be seen + // next to the nop instruction in disasm. + // e.g. D43E0000 align [4 bytes for IG07] + ins = INS_BREAKPOINT; + fmt = IF_SI_0A; + } +#endif } +#endif // FEATURE_LOOP_ALIGN - switch (ins) + if (!skipIns) { - case INS_sve_prfh: - code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii - break; - - case INS_sve_prfw: - code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii - break; + code = emitInsCode(ins, fmt); + dst += emitOutput_Instr(dst, code); + } - case INS_sve_prfd: - code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii - break; + break; + } - default: - assert(ins == INS_sve_prfb); - } + case IF_SI_0A: // SI_0A ...........iiiii iiiiiiiiiii..... imm16 + imm = emitGetInsSC(id); + assert(isValidUimm<16>(imm)); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 5); // iiiii iiiiiiiiiii dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HX_3A_B: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeUimm<20, 16>(imm); // iiiii - code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); + case IF_SI_0B: // SI_0B ................ ....bbbb........ imm4 - barrier + imm = emitGetInsSC(id); + assert((imm >= 0) && (imm <= 15)); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 8); // bbbb dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HX_3A_E: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) - case IF_SVE_IV_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit gather load (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); - - switch (ins) - { - case INS_sve_ld1d: - case INS_sve_ldff1d: - code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii - break; - - case INS_sve_ld1w: - case INS_sve_ld1sw: - case INS_sve_ldff1w: - case INS_sve_ldff1sw: - code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii - break; - - default: - code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii - break; - } - + case IF_SR_1A: // SR_1A ................ ...........ttttt Rt (dc zva, mrs) + assert(insOptsNone(id->idInsOpt())); + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Rt(id->idReg1()); // ttttt dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JL_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit scatter store (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii - dst += emitOutput_Instr(dst, code); + default: + dst = emitOutput_InstrSve(dst, id); break; + } - case IF_SVE_JI_3A_A: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit scatter store (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); + // Determine if any registers now hold GC refs, or whether a register that was overwritten held a GC ref. + // We assume here that "id->idGCref()" is not GC_NONE only if the instruction described by "id" writes a + // GC ref to register "id->idReg1()". (It may, apparently, also not be GC_NONE in other cases, such as + // for stores, but we ignore those cases here.) + if (emitInsMayWriteToGCReg(id)) // True if "id->idIns()" writes to a register than can hold GC ref. + { + // We assume that "idReg1" is the primary destination register for all instructions + assert(!emitInsDestIsOp2(ins)); + if (id->idGCref() != GCT_NONE) + { + emitGCregLiveUpd(id->idGCref(), id->idReg1(), dst); + } + else + { + emitGCregDeadUpd(id->idReg1(), dst); + } - switch (ins) + if (emitInsMayWriteMultipleRegs(id)) + { + // INS_ldp etc... + // "idReg2" is the secondary destination register + if (id->idGCrefReg2() != GCT_NONE) { - case INS_sve_st1h: - code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii - break; - - case INS_sve_st1w: - code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii - break; - - default: - assert(ins == INS_sve_st1b); - code |= insEncodeUimm<20, 16>(imm); // iiiii - break; + emitGCregLiveUpd(id->idGCrefReg2(), id->idReg2(), dst); + } + else + { + emitGCregDeadUpd(id->idReg2(), dst); } + } + } - dst += emitOutput_Instr(dst, code); - break; +SKIP_GC_UPDATE: + // Now we determine if the instruction has written to a (local variable) stack location, and either written a GC + // ref or overwritten one. + if (emitInsWritesToLclVarStackLoc(id) || emitInsWritesToLclVarStackLocPair(id)) + { + int varNum = id->idAddr()->iiaLclVar.lvaVarNum(); + unsigned ofs = AlignDown(id->idAddr()->iiaLclVar.lvaOffset(), TARGET_POINTER_SIZE); + bool FPbased; + int adr = emitComp->lvaFrameAddress(varNum, &FPbased); + if (id->idGCref() != GCT_NONE) + { + emitGCvarLiveUpd(adr + ofs, varNum, id->idGCref(), dst DEBUG_ARG(varNum)); + } + else + { + // If the type of the local is a gc ref type, update the liveness. + var_types vt; + if (varNum >= 0) + { + // "Regular" (non-spill-temp) local. + vt = var_types(emitComp->lvaTable[varNum].lvType); + } + else + { + TempDsc* tmpDsc = codeGen->regSet.tmpFindNum(varNum); + vt = tmpDsc->tdTempType(); + } + if (vt == TYP_REF || vt == TYP_BYREF) + { + emitGCvarDeadUpd(adr + ofs, dst DEBUG_ARG(varNum)); + } + } + if (emitInsWritesToLclVarStackLocPair(id)) + { + int varNum2 = varNum; + int adr2 = adr; + unsigned ofs2 = ofs; + unsigned ofs2Dist; - case IF_SVE_IA_2A: // ..........iiiiii ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= id->idSvePrfop(); // oooo - code |= insEncodeSimm<21, 16>(imm); // iiiiii - dst += emitOutput_Instr(dst, code); - break; + if (id->idIsLclVarPair()) + { + bool FPbased2; - case IF_SVE_IC_3A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + emitLclVarAddr* lclVarAddr2 = emitGetLclVarPairLclVar2(id); + varNum2 = lclVarAddr2->lvaVarNum(); + ofs2 = lclVarAddr2->lvaOffset(); - switch (ins) + // If there are 2 GC vars in this instrDesc, get the 2nd variable + // that should be tracked. + adr2 = emitComp->lvaFrameAddress(varNum2, &FPbased2); + ofs2Dist = EA_SIZE_IN_BYTES(size); +#ifdef DEBUG + assert(FPbased == FPbased2); + if (FPbased) + { + assert(id->idReg3() == REG_FP); + } + else + { + assert(id->idReg3() == REG_SP); + } + assert(varNum2 != -1); +#endif // DEBUG + } + else { - case INS_sve_ld1rd: - code |= insEncodeUimm6_MultipleOf8_21_to_16(imm); // iiiiii - break; - - default: - assert(ins == INS_sve_ld1rsw); - code |= insEncodeUimm6_MultipleOf4_21_to_16(imm); // iiiiii - break; + ofs2Dist = TARGET_POINTER_SIZE; + ofs2 += ofs2Dist; } - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_IC_3A_A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element - case IF_SVE_IC_3A_B: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element - case IF_SVE_IC_3A_C: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code = insEncodeSveElemsize_dtypeh_dtypel(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); + ofs2 = AlignDown(ofs2, ofs2Dist); - switch (ins) + if (id->idGCrefReg2() != GCT_NONE) { - case INS_sve_ld1rw: - code |= insEncodeUimm6_MultipleOf4_21_to_16(imm); // iiiiii - break; - - case INS_sve_ld1rh: - case INS_sve_ld1rsh: - code |= insEncodeUimm6_MultipleOf2_21_to_16(imm); // iiiiii - break; - - default: - code |= insEncodeUimm<21, 16>(imm); // iiiiii - break; +#ifdef DEBUG + if (id->idGCref() != GCT_NONE) + { + // If 1st register was a gc-var, then make sure the offset + // are correctly set for the 2nd register that is holding + // another gc-var. + assert((adr + ofs + ofs2Dist) == (adr2 + ofs2)); + } +#endif + emitGCvarLiveUpd(adr2 + ofs2, varNum2, id->idGCrefReg2(), dst DEBUG_ARG(varNum2)); + } + else + { + // If the type of the local is a gc ref type, update the liveness. + var_types vt; + if (varNum2 >= 0) + { + // "Regular" (non-spill-temp) local. + vt = var_types(emitComp->lvaTable[varNum2].lvType); + } + else + { + TempDsc* tmpDsc = codeGen->regSet.tmpFindNum(varNum2); + vt = tmpDsc->tdTempType(); + } + if (vt == TYP_REF || vt == TYP_BYREF) + { + emitGCvarDeadUpd(adr2 + ofs2, dst DEBUG_ARG(varNum2)); + } } + } + } - dst += emitOutput_Instr(dst, code); - break; +#ifdef DEBUG + /* Make sure we set the instruction descriptor size correctly */ - case IF_SVE_BI_2A: // ................ ......nnnnnddddd -- SVE constructive prefix (unpredicated) - case IF_SVE_HH_2A: // ................ ......nnnnnddddd -- SVE2 FP8 upconverts - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; + size_t expected = emitSizeOfInsDsc(id); + assert(sz == expected); - case IF_SVE_CB_2A: // ........xx...... ......nnnnnddddd -- SVE broadcast general register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + if (emitComp->opts.disAsm || emitComp->verbose) + { + emitDispIns(id, false, dspOffs, true, emitCurCodeOffs(odst), *dp, (dst - *dp), ig); + } - case IF_SVE_BJ_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point exponential accelerator - case IF_SVE_CG_2A: // ........xx...... ......nnnnnddddd -- SVE reverse vector elements - case IF_SVE_CH_2A: // ........xx...... ......nnnnnddddd -- SVE unpack vector elements - case IF_SVE_HF_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point reciprocal estimate (unpredicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + if (emitComp->compDebugBreak) + { + // For example, set JitBreakEmitOutputInstr=a6 will break when this method is called for + // emitting instruction a6, (i.e. IN00a6 in jitdump). + if ((unsigned)JitConfig.JitBreakEmitOutputInstr() == id->idDebugOnlyInfo()->idNum) + { + assert(!"JitBreakEmitOutputInstr reached"); + } + } - case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) - case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert - case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsizeWithShift_tszh_tszl_imm3(id->idInsOpt(), imm, - emitInsIsVectorRightShift(ins)); // xx xxiii - dst += emitOutput_Instr(dst, code); - break; + // Output any delta in GC info. + if (EMIT_GC_VERBOSE || emitComp->opts.disasmWithGC) + { + emitDispGCInfoDelta(); + } +#else + if (emitComp->opts.disAsm) + { + size_t expected = emitSizeOfInsDsc(id); + assert(sz == expected); + emitDispIns(id, false, 0, true, emitCurCodeOffs(odst), *dp, (dst - *dp), ig); + } +#endif - case IF_SVE_BX_2A: // ...........ixxxx ......nnnnnddddd -- sve_int_perm_dupq_i - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsizeWithImmediate_i1_tsz(id->idInsOpt(), imm); // ixxxx - dst += emitOutput_Instr(dst, code); - break; + /* All instructions are expected to generate code */ - case IF_SVE_BY_2A: // ............iiii ......mmmmmddddd -- sve_int_perm_extq - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm<19, 16>(imm); // iiii - dst += emitOutput_Instr(dst, code); - break; + assert(*dp != dst || id->idIsEmptyAlign()); - default: - assert(!"Unexpected format"); - break; - } + *dp = dst; - return dst; + return sz; } /*****************************************************************************/ @@ -20988,305 +14625,100 @@ void emitter::emitDispSmallFloatImm(ssize_t imm, instruction ins) /***************************************************************************** * - * Display an immediate with an optional left-shift. - */ -void emitter::emitDispImmOptsLSL(ssize_t imm, bool hasShift, unsigned shiftAmount) -{ - if (!strictArmAsm && hasShift) - { - imm <<= shiftAmount; - } - emitDispImm(imm, false); - if (strictArmAsm && hasShift) - { - printf(", LSL #%u", shiftAmount); - } -} - -/***************************************************************************** - * - * Display an ARM64 condition code for the conditional instructions - */ -void emitter::emitDispCond(insCond cond) -{ - const static char* armCond[16] = {"eq", "ne", "hs", "lo", "mi", "pl", "vs", "vc", - "hi", "ls", "ge", "lt", "gt", "le", "AL", "NV"}; // The last two are invalid - unsigned imm = (unsigned)cond; - assert((0 <= imm) && (imm < ArrLen(armCond))); - printf(armCond[imm]); -} - -/***************************************************************************** - * - * Display an ARM64 flags for the conditional instructions - */ -void emitter::emitDispFlags(insCflags flags) -{ - const static char* armFlags[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv", - "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"}; - unsigned imm = (unsigned)flags; - assert((0 <= imm) && (imm < ArrLen(armFlags))); - printf(armFlags[imm]); -} - -/***************************************************************************** - * - * Display an ARM64 'barrier' for the memory barrier instructions - */ -void emitter::emitDispBarrier(insBarrier barrier) -{ - const static char* armBarriers[16] = {"#0", "oshld", "oshst", "osh", "#4", "nshld", "nshst", "nsh", - "#8", "ishld", "ishst", "ish", "#12", "ld", "st", "sy"}; - unsigned imm = (unsigned)barrier; - assert((0 <= imm) && (imm < ArrLen(armBarriers))); - printf(armBarriers[imm]); -} - -/***************************************************************************** - * - * Prints the encoding for the Shift Type encoding - */ - -void emitter::emitDispShiftOpts(insOpts opt) -{ - if (opt == INS_OPTS_LSL) - printf(" LSL "); - else if (opt == INS_OPTS_LSR) - printf(" LSR "); - else if (opt == INS_OPTS_ASR) - printf(" ASR "); - else if (opt == INS_OPTS_ROR) - printf(" ROR "); - else if (opt == INS_OPTS_MSL) - printf(" MSL "); - else - assert(!"Bad value"); -} - -/***************************************************************************** - * - * Prints the encoding for the Extend Type encoding - */ - -void emitter::emitDispExtendOpts(insOpts opt) -{ - if (opt == INS_OPTS_UXTB) - printf("UXTB"); - else if (opt == INS_OPTS_UXTH) - printf("UXTH"); - else if (opt == INS_OPTS_UXTW) - printf("UXTW"); - else if (opt == INS_OPTS_UXTX) - printf("UXTX"); - else if (opt == INS_OPTS_SXTB) - printf("SXTB"); - else if (opt == INS_OPTS_SXTH) - printf("SXTH"); - else if (opt == INS_OPTS_SXTW) - printf("SXTW"); - else if (opt == INS_OPTS_SXTX) - printf("SXTX"); - else - assert(!"Bad value"); -} - -/***************************************************************************** - * - * Prints the encoding for the Extend Type encoding - */ - -void emitter::emitDispSveExtendOpts(insOpts opt) -{ - switch (opt) - { - case INS_OPTS_LSL: - printf("lsl"); - break; - - case INS_OPTS_UXTW: - case INS_OPTS_SCALABLE_S_UXTW: - case INS_OPTS_SCALABLE_D_UXTW: - printf("uxtw"); - break; - - case INS_OPTS_SXTW: - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - printf("sxtw"); - break; - - default: - assert(!"Bad value"); - break; - } -} - -/***************************************************************************** - * - * Prints the encoding for the Extend Type encoding along with the N value - */ - -void emitter::emitDispSveExtendOptsModN(insOpts opt, ssize_t imm) -{ - assert(imm >= 0 && imm <= 3); - - if (imm == 0 && opt != INS_OPTS_LSL) - { - emitDispSveExtendOpts(opt); - } - else if (imm > 0) - { - emitDispSveExtendOpts(opt); - printf(" #%d", (int)imm); - } -} - -/***************************************************************************** - * - * Prints the encoding for the or LSL encoding along with the N value - * This is for formats that have [, .T, ], [, .T, #N], [, , LSL #N], - * [{, , LSL #N}] + * Display an immediate with an optional left-shift. */ -void emitter::emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2, insOpts opt, insFormat fmt) +void emitter::emitDispImmOptsLSL(ssize_t imm, bool hasShift, unsigned shiftAmount) { - printf("["); - - if (isVectorRegister(reg1)) - { - // If the overall instruction is working on 128-bit - // registers, the size of this register for - // the mod addr is always 64-bit. - // Example: LD1Q {.Q }, /Z, [.D{, }] - if (opt == INS_OPTS_SCALABLE_Q) - { - emitDispSveReg(reg1, INS_OPTS_SCALABLE_D, reg2 != REG_ZR); - } - else - { - emitDispSveReg(reg1, opt, reg2 != REG_ZR); - } - } - else - { - emitDispReg(reg1, EA_8BYTE, reg2 != REG_ZR); - } - - if (isVectorRegister(reg2)) - { - emitDispSveReg(reg2, opt, false); - } - else if (reg2 != REG_ZR) + if (!strictArmAsm && hasShift) { - emitDispReg(reg2, EA_8BYTE, false); + imm <<= shiftAmount; } - - if (insOptsScalable32bitExtends(opt)) + emitDispImm(imm, false); + if (strictArmAsm && hasShift) { - emitDispComma(); - emitDispSveExtendOptsModN(opt, insSveGetLslOrModN(ins, fmt)); + printf(", LSL #%u", shiftAmount); } - // Omit 'lsl #N' only if the second register is ZR. - else if ((reg2 != REG_ZR) && insSveIsLslN(ins, fmt)) - { - emitDispComma(); - switch (insSveGetLslOrModN(ins, fmt)) - { - case 4: - printf("lsl #4"); - break; - - case 3: - printf("lsl #3"); - break; - - case 2: - printf("lsl #2"); - break; - - case 1: - printf("lsl #1"); - break; +} - default: - assert(!"Invalid instruction"); - break; - } - } - printf("]"); +/***************************************************************************** + * + * Display an ARM64 condition code for the conditional instructions + */ +void emitter::emitDispCond(insCond cond) +{ + const static char* armCond[16] = {"eq", "ne", "hs", "lo", "mi", "pl", "vs", "vc", + "hi", "ls", "ge", "lt", "gt", "le", "AL", "NV"}; // The last two are invalid + unsigned imm = (unsigned)cond; + assert((0 <= imm) && (imm < ArrLen(armCond))); + printf(armCond[imm]); } /***************************************************************************** * - * Prints the encoding for format [.S{, #}] + * Display an ARM64 flags for the conditional instructions */ -void emitter::emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt) +void emitter::emitDispFlags(insCflags flags) { - printf("["); - emitDispSveReg(reg1, opt, imm != 0); - if (imm != 0) - { - // This does not have to be printed as hex. - // We only do it because the capstone disassembly displays this immediate as hex. - // We could not modify capstone without affecting other cases. - emitDispImm(imm, false, /* alwaysHex */ true); - } - printf("]"); + const static char* armFlags[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv", + "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"}; + unsigned imm = (unsigned)flags; + assert((0 <= imm) && (imm < ArrLen(armFlags))); + printf(armFlags[imm]); } /***************************************************************************** * - * Prints the encoding for format [{, #, MUL VL}] + * Display an ARM64 'barrier' for the memory barrier instructions */ -void emitter::emitDispSveImmMulVl(regNumber reg1, ssize_t imm) +void emitter::emitDispBarrier(insBarrier barrier) { - printf("["); - emitDispReg(reg1, EA_8BYTE, imm != 0); - if (imm != 0) - { - emitDispImm(imm, true); - printf("mul vl"); - } - printf("]"); + const static char* armBarriers[16] = {"#0", "oshld", "oshst", "osh", "#4", "nshld", "nshst", "nsh", + "#8", "ishld", "ishst", "ish", "#12", "ld", "st", "sy"}; + unsigned imm = (unsigned)barrier; + assert((0 <= imm) && (imm < ArrLen(armBarriers))); + printf(armBarriers[imm]); } /***************************************************************************** * - * Prints the encoding for format [.D{, #}] + * Prints the encoding for the Shift Type encoding */ -void emitter::emitDispSveImmIndex(regNumber reg1, insOpts opt, ssize_t imm) + +void emitter::emitDispShiftOpts(insOpts opt) { - printf("["); - if (isVectorRegister(reg1)) - { - emitDispSveReg(reg1, opt, imm != 0); - } + if (opt == INS_OPTS_LSL) + printf(" LSL "); + else if (opt == INS_OPTS_LSR) + printf(" LSR "); + else if (opt == INS_OPTS_ASR) + printf(" ASR "); + else if (opt == INS_OPTS_ROR) + printf(" ROR "); + else if (opt == INS_OPTS_MSL) + printf(" MSL "); else - { - emitDispReg(reg1, EA_8BYTE, imm != 0); - } - if (imm != 0) - { - // This does not have to be printed as hex. - // We only do it because the capstone disassembly displays this immediate as hex. - // We could not modify capstone without affecting other cases. - emitDispImm(imm, false, /* alwaysHex */ (imm > 31)); - } - printf("]"); + assert(!"Bad value"); } /***************************************************************************** * - * Prints the encoding for the Extend Type encoding in loads/stores + * Prints the encoding for the Extend Type encoding */ -void emitter::emitDispLSExtendOpts(insOpts opt) +void emitter::emitDispExtendOpts(insOpts opt) { - if (opt == INS_OPTS_LSL) - printf("LSL"); + if (opt == INS_OPTS_UXTB) + printf("UXTB"); + else if (opt == INS_OPTS_UXTH) + printf("UXTH"); else if (opt == INS_OPTS_UXTW) printf("UXTW"); else if (opt == INS_OPTS_UXTX) printf("UXTX"); + else if (opt == INS_OPTS_SXTB) + printf("SXTB"); + else if (opt == INS_OPTS_SXTH) + printf("SXTH"); else if (opt == INS_OPTS_SXTW) printf("SXTW"); else if (opt == INS_OPTS_SXTX) @@ -21307,46 +14739,6 @@ void emitter::emitDispReg(regNumber reg, emitAttr attr, bool addComma) emitDispComma(); } -//------------------------------------------------------------------------ -// emitDispSveReg: Display a scalable vector register name -// -void emitter::emitDispSveReg(regNumber reg, bool addComma) -{ - assert(isVectorRegister(reg)); - printf(emitSveRegName(reg)); - - if (addComma) - emitDispComma(); -} - -//------------------------------------------------------------------------ -// emitDispSveReg: Display a scalable vector register name with an arrangement suffix -// -void emitter::emitDispSveReg(regNumber reg, insOpts opt, bool addComma) -{ - assert(isVectorRegister(reg)); - printf(emitSveRegName(reg)); - - if (opt != INS_OPTS_NONE) - { - assert(insOptsScalable(opt) || insOptsScalable32bitExtends(opt)); - emitDispArrangement(opt); - } - - if (addComma) - emitDispComma(); -} - -//------------------------------------------------------------------------ -// emitDispSveRegIndex: Display a scalable vector register with indexed element -// -void emitter::emitDispSveRegIndex(regNumber reg, ssize_t index, bool addComma) -{ - assert(isVectorRegister(reg)); - printf(emitSveRegName(reg)); - emitDispElementIndex(index, addComma); -} - //------------------------------------------------------------------------ // emitDispVectorReg: Display a SIMD vector register name with an arrangement suffix // @@ -21429,122 +14821,6 @@ void emitter::emitDispVectorElemList( } } -//------------------------------------------------------------------------ -// emitDispSveConsecutiveRegList: Display a SVE consecutive vector register list -// -void emitter::emitDispSveConsecutiveRegList(regNumber firstReg, unsigned listSize, insOpts opt, bool addComma) -{ - assert(isVectorRegister(firstReg)); - - regNumber currReg = firstReg; - - assert(listSize > 0); - - printf("{ "); - // We do not want the short-hand for list size of 1 or 2. - if ((listSize <= 2) || (((unsigned)currReg + listSize - 1) > (unsigned)REG_V31)) - { - for (unsigned i = 0; i < listSize; i++) - { - const bool notLastRegister = (i != listSize - 1); - emitDispSveReg(currReg, opt, notLastRegister); - currReg = (currReg == REG_V31) ? REG_V0 : REG_NEXT(currReg); - } - } - else - { - // short-hand. example: { z0.s - z2.s } which is the same as { z0.s, z1.s, z2.s } - emitDispSveReg(currReg, opt, false); - printf(" - "); - emitDispSveReg((regNumber)(currReg + listSize - 1), opt, false); - } - printf(" }"); - - if (addComma) - { - emitDispComma(); - } -} - -//------------------------------------------------------------------------ -// emitDispPredicateReg: Display a predicate register name with with an arrangement suffix -// -void emitter::emitDispPredicateReg(regNumber reg, PredicateType ptype, insOpts opt, bool addComma) -{ - assert(isPredicateRegister(reg)); - printf(emitPredicateRegName(reg, ptype)); - - if (ptype == PREDICATE_MERGE) - { - printf("/m"); - } - else if (ptype == PREDICATE_ZERO) - { - printf("/z"); - } - else if (ptype == PREDICATE_SIZED || ptype == PREDICATE_N_SIZED) - { - emitDispElemsize(optGetSveElemsize(opt)); - } - - if (addComma) - emitDispComma(); -} - -//------------------------------------------------------------------------ -// emitDispPredicateRegPair: Display a pair of predicate registers -// -void emitter::emitDispPredicateRegPair(regNumber reg, insOpts opt) -{ - printf("{ "); - emitDispPredicateReg(reg, PREDICATE_SIZED, opt, true); - emitDispPredicateReg((regNumber)((unsigned)reg + 1), PREDICATE_SIZED, opt, false); - printf(" }, "); -} - -//------------------------------------------------------------------------ -// emitDispLowPredicateReg: Display a low predicate register name with with an arrangement suffix -// -void emitter::emitDispLowPredicateReg(regNumber reg, PredicateType ptype, insOpts opt, bool addComma) -{ - assert(isLowPredicateRegister(reg)); - reg = (regNumber)((((unsigned)reg - REG_PREDICATE_FIRST) & 0x7) + REG_PREDICATE_FIRST); - emitDispPredicateReg(reg, ptype, opt, addComma); -} - -//------------------------------------------------------------------------ -// emitDispLowPredicateRegPair: Display a pair of low predicate registers -// -void emitter::emitDispLowPredicateRegPair(regNumber reg, insOpts opt) -{ - assert(isLowPredicateRegister(reg)); - - printf("{ "); - const unsigned baseRegNum = ((unsigned)reg - REG_PREDICATE_FIRST) & 0x7; - const unsigned regNum = (baseRegNum * 2) + REG_PREDICATE_FIRST; - emitDispPredicateReg((regNumber)regNum, PREDICATE_SIZED, opt, true); - emitDispPredicateReg((regNumber)(regNum + 1), PREDICATE_SIZED, opt, false); - printf(" }, "); -} - -//------------------------------------------------------------------------ -// emitDispVectorLengthSpecifier: Display the vector length specifier -// -void emitter::emitDispVectorLengthSpecifier(instrDesc* id) -{ - assert(id != nullptr); - assert(insOptsScalableStandard(id->idInsOpt())); - - if (id->idVectorLength4x()) - { - printf("vlx4"); - } - else - { - printf("vlx2"); - } -} - //------------------------------------------------------------------------ // emitDispArrangement: Display a SIMD vector arrangement suffix // @@ -21599,7 +14875,7 @@ void emitter::emitDispArrangement(insOpts opt) break; default: - assert(!"Invalid SVE insOpt"); + assert(!"Invalid insOpt"); } printf("."); printf(str); @@ -21814,103 +15090,6 @@ void emitter::emitDispAddrRRExt(regNumber reg1, regNumber reg2, insOpts opt, boo printf("]"); } -/***************************************************************************** - * - * Display an insSvePattern - */ -void emitter::emitDispSvePattern(insSvePattern pattern, bool addComma) -{ - printf("%s", svePatternNames[pattern]); - - if (addComma) - { - emitDispComma(); - } -} - -/***************************************************************************** - * - * Display an insSvePrfop - */ -void emitter::emitDispSvePrfop(insSvePrfop prfop, bool addComma) -{ - switch (prfop) - { - case SVE_PRFOP_PLDL1KEEP: - printf("pldl1keep"); - break; - - case SVE_PRFOP_PLDL1STRM: - printf("pldl1strm"); - break; - - case SVE_PRFOP_PLDL2KEEP: - printf("pldl2keep"); - break; - - case SVE_PRFOP_PLDL2STRM: - printf("pldl2strm"); - break; - - case SVE_PRFOP_PLDL3KEEP: - printf("pldl3keep"); - break; - - case SVE_PRFOP_PLDL3STRM: - printf("pldl3strm"); - break; - - case SVE_PRFOP_PSTL1KEEP: - printf("pstl1keep"); - break; - - case SVE_PRFOP_PSTL1STRM: - printf("pstl1strm"); - break; - - case SVE_PRFOP_PSTL2KEEP: - printf("pstl2keep"); - break; - - case SVE_PRFOP_PSTL2STRM: - printf("pstl2strm"); - break; - - case SVE_PRFOP_PSTL3KEEP: - printf("pstl3keep"); - break; - - case SVE_PRFOP_PSTL3STRM: - printf("pstl3strm"); - break; - - case SVE_PRFOP_CONST6: - printf("#6"); - break; - - case SVE_PRFOP_CONST7: - printf("#7"); - break; - - case SVE_PRFOP_CONST14: - printf("#0xE"); - break; - - case SVE_PRFOP_CONST15: - printf("#0xF"); - break; - - default: - assert(!"Invalid prfop"); - break; - } - - if (addComma) - { - emitDispComma(); - } -} - /***************************************************************************** * * Display (optionally) the instruction encoding in hex @@ -23381,7 +16560,7 @@ void emitter::emitDispInsHelp( { ssize_t imm1; ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd emitDispImm(imm1, true); // iiiii emitDispImm(imm2, false); // iiiii diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 01a038c8a4dccf..0ddf5a86c803c2 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -29,7 +29,7 @@ enum PredicateType PREDICATE_N_SIZED, // Predicate printed as counter with element size }; -const char* emitSveRegName(regNumber reg); +const char* emitSveRegName(regNumber reg) const; const char* emitVectorRegName(regNumber reg); const char* emitPredicateRegName(regNumber reg, PredicateType ptype); @@ -56,7 +56,6 @@ void emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2, insOpts void emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt); void emitDispSveImmMulVl(regNumber reg1, ssize_t imm); void emitDispSveImmIndex(regNumber reg1, insOpts opt, ssize_t imm); -void emitDispLSExtendOpts(insOpts opt); void emitDispReg(regNumber reg, emitAttr attr, bool addComma); void emitDispSveReg(regNumber reg, bool addComma); void emitDispSveReg(regNumber reg, insOpts opt, bool addComma); @@ -220,8 +219,6 @@ union bitMaskImm { * representation imm(i16,hw) */ -static emitter::bitMaskImm emitEncodeBitMaskImm(INT64 imm, emitAttr size); - static INT64 emitDecodeBitMaskImm(const emitter::bitMaskImm bmImm, emitAttr size); /************************************************************************ @@ -245,8 +242,6 @@ union halfwordImm { * representation imm(i16,hw) */ -static emitter::halfwordImm emitEncodeHalfwordImm(INT64 imm, emitAttr size); - static INT64 emitDecodeHalfwordImm(const emitter::halfwordImm hwImm, emitAttr size); /************************************************************************ @@ -271,8 +266,6 @@ union byteShiftedImm { * representation imm(i8,by) */ -static emitter::byteShiftedImm emitEncodeByteShiftedImm(INT64 imm, emitAttr size, bool allow_MSL); - static UINT32 emitDecodeByteShiftedImm(const emitter::byteShiftedImm bsImm, emitAttr size); /************************************************************************ @@ -296,8 +289,6 @@ union floatImm8 { * Convert between a double and its 'float 8-bit immediate' representation */ -static emitter::floatImm8 emitEncodeFloatImm8(double immDbl); - static double emitDecodeFloatImm8(const emitter::floatImm8 fpImm); static ssize_t emitEncodeRotationImm90_or_270(ssize_t imm); @@ -741,14 +732,14 @@ static code_t insEncodeSveShift_23_to_22_9_to_0(emitAttr size, bool isRightShift // for an Arm64 Sve instruction. static code_t insEncodeSveElemsize_R_22(emitAttr size); -// Returns the immediate value for instructions that encode it as a difference from tszh:tszl:imm3. -static ssize_t insGetImmDiff(const ssize_t imm, const insOpts opt); +// Returns the immediate value for SVE instructions that encode it as a difference from tszh:tszl:imm3. +static ssize_t insSveGetImmDiff(const ssize_t imm, const insOpts opt); // Returns the two 5-bit signed immediates encoded as one ssize_t. -static ssize_t insEncodeTwoSimm5(ssize_t imm1, ssize_t imm2); +static ssize_t insSveEncodeTwoSimm5(ssize_t imm1, ssize_t imm2); -// Decodes imm into two 5-bit signed immediates, using the encoding format from insEncodeTwoSimm5. -static void insDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2); +// Decodes imm into two 5-bit signed immediates, using the encoding format from insSveEncodeTwoSimm5. +static void insSveDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2); // Returns the encoding to select an insSvePattern static code_t insEncodeSvePattern(insSvePattern pattern); @@ -1054,9 +1045,6 @@ static bool canEncodeWithShiftImmBy12(INT64 imm); // Normalize the 'imm' so that the upper bits, as defined by 'size' are zero static INT64 normalizeImm64(INT64 imm, emitAttr size); -// Normalize the 'imm' so that the upper bits, as defined by 'size' are zero -static INT32 normalizeImm32(INT32 imm, emitAttr size); - // true if 'imm' can be encoded using a 'bitmask immediate', also returns the encoding if wbBMI is non-null static bool canEncodeBitMaskImm(INT64 imm, emitAttr size, emitter::bitMaskImm* wbBMI = nullptr); diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 66f4cebd61cd8e..577d815379dbed 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -22,1320 +22,1507 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "instr.h" -/***************************************************************************** - * - * Add a SVE instruction with a single immediate value. - */ +/*****************************************************************************/ -void emitter::emitInsSve_I(instruction ins, emitAttr attr, ssize_t imm) +// clang-format off +static const char * const zRegNames[] = { - insFormat fmt; - - /* Figure out the encoding format of the instruction */ - if (ins == INS_sve_setffr) - { - fmt = IF_SVE_DQ_0A; - attr = EA_PTRSIZE; - imm = 0; - } - else - { - unreached(); - } - - instrDesc* id = emitNewInstrSC(attr, imm); + "z0", "z1", "z2", "z3", "z4", + "z5", "z6", "z7", "z8", "z9", + "z10", "z11", "z12", "z13", "z14", + "z15", "z16", "z17", "z18", "z19", + "z20", "z21", "z22", "z23", "z24", + "z25", "z26", "z27", "z28", "z29", + "z30", "z31" +}; + +static const char * const pRegNames[] = +{ + "p0", "p1", "p2", "p3", "p4", + "p5", "p6", "p7", "p8", "p9", + "p10", "p11", "p12", "p13", "p14", + "p15" +}; - id->idIns(ins); - id->idInsFmt(fmt); +static const char * const pnRegNames[] = +{ + "pn0", "pn1", "pn2", "pn3", "pn4", + "pn5", "pn6", "pn7", "pn8", "pn9", + "pn10", "pn11", "pn12", "pn13", "pn14", + "pn15" +}; - dispIns(id); - appendToCurIG(id); -} +static const char * const svePatternNames[] = +{ + "pow2", "vl1", "vl2", "vl3", + "vl4", "vl5", "vl6", "vl7", + "vl8", "vl16", "vl32", "vl64", + "vl128", "vl256", "invalid", "invalid", + "invalid", "invalid", "invalid", "invalid", + "invalid", "invalid", "invalid", "invalid", + "invalid", "invalid", "invalid", "invalid", + "invalid", "mul4", "mul3", "all" +}; + +// clang-format on /***************************************************************************** * - * Add a SVE instruction referencing a single register. + * Returns the specific encoding of the given CPU instruction and format */ -void emitter::emitInsSve_R(instruction ins, emitAttr attr, regNumber reg, insOpts opt /* = INS_OPTS_NONE */) +emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) { - insFormat fmt; - - /* Figure out the encoding format of the instruction */ - switch (ins) + // clang-format off + const static code_t insCodes1[] = { - case INS_sve_aesmc: - case INS_sve_aesimc: - opt = INS_OPTS_SCALABLE_B; - assert(isVectorRegister(reg)); // ddddd - assert(isScalableVectorSize(attr)); - fmt = IF_SVE_GL_1A; + #define INST1(id, nm, info, fmt, e1 ) e1, + #define INST2(id, nm, info, fmt, e1, e2 ) e1, + #define INST3(id, nm, info, fmt, e1, e2, e3 ) e1, + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e1, + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e1, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e1, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e1, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e1, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e1, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e1, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e1, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes2[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) e2, + #define INST3(id, nm, info, fmt, e1, e2, e3 ) e2, + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e2, + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e2, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e2, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e2, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e2, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e2, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e2, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e2, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes3[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) e3, + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e3, + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e3, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e3, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e3, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e3, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e3, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e3, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e3, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes4[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e4, + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e4, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e4, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e4, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e4, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e4, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e4, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e4, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes5[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e5, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e5, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e5, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e5, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e5, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e5, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e5, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes6[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e6, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e6, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e6, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e6, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e6, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e6, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes7[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e7, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e7, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e7, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e7, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e7, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes8[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e8, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e8, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e8, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e8, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes9[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e9, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e9, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e9, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes10[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e10, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e10, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes11[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e11, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e11, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes12[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e12, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes13[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e13, + #include "instrsarm64sve.h" + }; + + // clang-format on + const static insFormat formatEncode13A[13] = {IF_SVE_AU_3A, IF_SVE_BT_1A, IF_SVE_BV_2A, IF_SVE_BV_2A_J, + IF_SVE_BW_2A, IF_SVE_CB_2A, IF_SVE_CP_3A, IF_SVE_CQ_3A, + IF_SVE_CW_4A, IF_SVE_CZ_4A, IF_SVE_CZ_4A_K, IF_SVE_CZ_4A_L, + IF_SVE_EB_1A}; + const static insFormat formatEncode11A[11] = {IF_SVE_JD_4B, IF_SVE_JD_4C, IF_SVE_JI_3A_A, IF_SVE_JJ_4A, + IF_SVE_JJ_4A_B, IF_SVE_JJ_4A_C, IF_SVE_JJ_4A_D, IF_SVE_JJ_4B, + IF_SVE_JJ_4B_E, IF_SVE_JN_3B, IF_SVE_JN_3C}; + const static insFormat formatEncode9A[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, + IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, + IF_SVE_HX_3A_E, IF_SVE_IJ_3A_F, IF_SVE_IK_4A_G}; + const static insFormat formatEncode9B[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, + IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, + IF_SVE_HX_3A_E, IF_SVE_IJ_3A_G, IF_SVE_IK_4A_I}; + const static insFormat formatEncode9C[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, + IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, + IF_SVE_HX_3A_E, IF_SVE_IH_3A_F, IF_SVE_II_4A_H}; + const static insFormat formatEncode9D[9] = {IF_SVE_IH_3A, IF_SVE_IH_3A_A, IF_SVE_II_4A, + IF_SVE_II_4A_B, IF_SVE_IU_4A, IF_SVE_IU_4A_C, + IF_SVE_IU_4B, IF_SVE_IU_4B_D, IF_SVE_IV_3A}; + const static insFormat formatEncode9E[9] = {IF_SVE_JD_4A, IF_SVE_JI_3A_A, IF_SVE_JJ_4A, + IF_SVE_JJ_4A_B, IF_SVE_JJ_4A_C, IF_SVE_JJ_4A_D, + IF_SVE_JJ_4B, IF_SVE_JJ_4B_E, IF_SVE_JN_3A}; + const static insFormat formatEncode9F[9] = {IF_SVE_JD_4C, IF_SVE_JD_4C_A, IF_SVE_JJ_4A, + IF_SVE_JJ_4A_B, IF_SVE_JJ_4B, IF_SVE_JJ_4B_C, + IF_SVE_JL_3A, IF_SVE_JN_3C, IF_SVE_JN_3C_D}; + const static insFormat formatEncode8A[8] = {IF_SVE_CE_2A, IF_SVE_CE_2B, IF_SVE_CE_2C, IF_SVE_CE_2D, + IF_SVE_CF_2A, IF_SVE_CF_2B, IF_SVE_CF_2C, IF_SVE_CF_2D}; + const static insFormat formatEncode8B[8] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, IF_SVE_HW_4A_C, + IF_SVE_HW_4B, IF_SVE_HW_4B_D, IF_SVE_HX_3A_E, IF_SVE_IG_4A_F}; + const static insFormat formatEncode8C[8] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, IF_SVE_HW_4A_C, + IF_SVE_HW_4B, IF_SVE_HW_4B_D, IF_SVE_HX_3A_E, IF_SVE_IG_4A_G}; + const static insFormat formatEncode7A[7] = {IF_SVE_IJ_3A, IF_SVE_IK_4A, IF_SVE_IU_4A, IF_SVE_IU_4A_A, + IF_SVE_IU_4B, IF_SVE_IU_4B_B, IF_SVE_IV_3A}; + const static insFormat formatEncode6A[6] = {IF_SVE_AE_3A, IF_SVE_BD_3A, IF_SVE_EE_1A, + IF_SVE_FD_3A, IF_SVE_FD_3B, IF_SVE_FD_3C}; + const static insFormat formatEncode6B[6] = {IF_SVE_GY_3A, IF_SVE_GY_3B, IF_SVE_GY_3B_D, + IF_SVE_HA_3A, IF_SVE_HA_3A_E, IF_SVE_HA_3A_F}; + const static insFormat formatEncode6C[6] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, + IF_SVE_HX_3A_B, IF_SVE_IJ_3A_D, IF_SVE_IK_4A_F}; + const static insFormat formatEncode6D[6] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, + IF_SVE_HX_3A_B, IF_SVE_IJ_3A_E, IF_SVE_IK_4A_H}; + const static insFormat formatEncode6E[6] = {IF_SVE_HY_3A, IF_SVE_HY_3A_A, IF_SVE_HY_3B, + IF_SVE_HZ_2A_B, IF_SVE_IA_2A, IF_SVE_IB_3A}; + const static insFormat formatEncode6F[6] = {IF_SVE_IG_4A, IF_SVE_IU_4A, IF_SVE_IU_4A_A, + IF_SVE_IU_4B, IF_SVE_IU_4B_B, IF_SVE_IV_3A}; + const static insFormat formatEncode6G[6] = {IF_SVE_JD_4A, IF_SVE_JI_3A_A, IF_SVE_JK_4A, + IF_SVE_JK_4A_B, IF_SVE_JK_4B, IF_SVE_JN_3A}; + const static insFormat formatEncode5A[5] = {IF_SVE_AM_2A, IF_SVE_AN_3A, IF_SVE_AO_3A, IF_SVE_BF_2A, IF_SVE_BG_3A}; + const static insFormat formatEncode5B[5] = {IF_SVE_GX_3A, IF_SVE_GX_3B, IF_SVE_HK_3A, IF_SVE_HL_3A, IF_SVE_HM_2A}; + const static insFormat formatEncode5C[5] = {IF_SVE_EF_3A, IF_SVE_EG_3A, IF_SVE_EH_3A, IF_SVE_EY_3A, IF_SVE_EY_3B}; + const static insFormat formatEncode5D[5] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, IF_SVE_HX_3A_B, + IF_SVE_IG_4A_D}; + const static insFormat formatEncode5E[5] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, IF_SVE_HX_3A_B, + IF_SVE_IG_4A_E}; + const static insFormat formatEncode4A[4] = {IF_SVE_AA_3A, IF_SVE_AU_3A, IF_SVE_BS_1A, IF_SVE_CZ_4A}; + const static insFormat formatEncode4B[4] = {IF_SVE_BU_2A, IF_SVE_BV_2B, IF_SVE_EA_1A, IF_SVE_EB_1B}; + const static insFormat formatEncode4E[4] = {IF_SVE_BE_3A, IF_SVE_FI_3A, IF_SVE_FI_3B, IF_SVE_FI_3C}; + const static insFormat formatEncode4F[4] = {IF_SVE_EM_3A, IF_SVE_FK_3A, IF_SVE_FK_3B, IF_SVE_FK_3C}; + const static insFormat formatEncode4G[4] = {IF_SVE_AR_4A, IF_SVE_FF_3A, IF_SVE_FF_3B, IF_SVE_FF_3C}; + const static insFormat formatEncode4H[4] = {IF_SVE_GM_3A, IF_SVE_GN_3A, IF_SVE_GZ_3A, IF_SVE_HB_3A}; + const static insFormat formatEncode4I[4] = {IF_SVE_AX_1A, IF_SVE_AY_2A, IF_SVE_AZ_2A, IF_SVE_BA_3A}; + const static insFormat formatEncode4J[4] = {IF_SVE_BV_2A, IF_SVE_BV_2A_A, IF_SVE_CP_3A, IF_SVE_CQ_3A}; + const static insFormat formatEncode4K[4] = {IF_SVE_IF_4A, IF_SVE_IF_4A_A, IF_SVE_IM_3A, IF_SVE_IN_4A}; + const static insFormat formatEncode4L[4] = {IF_SVE_IZ_4A, IF_SVE_IZ_4A_A, IF_SVE_JB_4A, IF_SVE_JM_3A}; + const static insFormat formatEncode3A[3] = {IF_SVE_AB_3A, IF_SVE_AT_3A, IF_SVE_EC_1A}; + const static insFormat formatEncode3B[3] = {IF_SVE_BH_3A, IF_SVE_BH_3B, IF_SVE_BH_3B_A}; + const static insFormat formatEncode3C[3] = {IF_SVE_BW_2A, IF_SVE_CB_2A, IF_SVE_EB_1A}; + const static insFormat formatEncode3D[3] = {IF_SVE_BR_3A, IF_SVE_BR_3B, IF_SVE_CI_3A}; + const static insFormat formatEncode3E[3] = {IF_SVE_AT_3A, IF_SVE_EC_1A, IF_SVE_ET_3A}; + const static insFormat formatEncode3F[3] = {IF_SVE_GU_3A, IF_SVE_GU_3B, IF_SVE_HU_4A}; + const static insFormat formatEncode3G[3] = {IF_SVE_GH_3A, IF_SVE_GH_3B, IF_SVE_GH_3B_B}; + const static insFormat formatEncode3H[3] = {IF_SVE_HK_3A, IF_SVE_HL_3A, IF_SVE_HM_2A}; + const static insFormat formatEncode3I[3] = {IF_SVE_CM_3A, IF_SVE_CN_3A, IF_SVE_CO_3A}; + const static insFormat formatEncode3J[3] = {IF_SVE_CX_4A, IF_SVE_CX_4A_A, IF_SVE_CY_3A}; + const static insFormat formatEncode3K[3] = {IF_SVE_CX_4A, IF_SVE_CX_4A_A, IF_SVE_CY_3B}; + const static insFormat formatEncode3L[3] = {IF_SVE_DT_3A, IF_SVE_DX_3A, IF_SVE_DY_3A}; + const static insFormat formatEncode3M[3] = {IF_SVE_EJ_3A, IF_SVE_FA_3A, IF_SVE_FA_3B}; + const static insFormat formatEncode3N[3] = {IF_SVE_EK_3A, IF_SVE_FB_3A, IF_SVE_FB_3B}; + const static insFormat formatEncode3O[3] = {IF_SVE_EK_3A, IF_SVE_FC_3A, IF_SVE_FC_3B}; + const static insFormat formatEncode3P[3] = {IF_SVE_EL_3A, IF_SVE_FG_3A, IF_SVE_FG_3B}; + const static insFormat formatEncode3Q[3] = {IF_SVE_EO_3A, IF_SVE_FJ_3A, IF_SVE_FJ_3B}; + const static insFormat formatEncode3R[3] = {IF_SVE_FE_3A, IF_SVE_FE_3B, IF_SVE_FN_3A}; + const static insFormat formatEncode3S[3] = {IF_SVE_FH_3A, IF_SVE_FH_3B, IF_SVE_FN_3A}; + const static insFormat formatEncode3T[3] = {IF_SVE_GX_3C, IF_SVE_HK_3B, IF_SVE_HL_3B}; + const static insFormat formatEncode3U[3] = {IF_SVE_IM_3A, IF_SVE_IN_4A, IF_SVE_IX_4A}; + const static insFormat formatEncode3V[3] = {IF_SVE_JA_4A, IF_SVE_JB_4A, IF_SVE_JM_3A}; + const static insFormat formatEncode2AA[2] = {IF_SVE_ID_2A, IF_SVE_IE_2A}; + const static insFormat formatEncode2AB[2] = {IF_SVE_JG_2A, IF_SVE_JH_2A}; + const static insFormat formatEncode2AC[2] = {IF_SVE_AD_3A, IF_SVE_ED_1A}; + const static insFormat formatEncode2AD[2] = {IF_SVE_AB_3B, IF_SVE_AT_3B}; + const static insFormat formatEncode2AE[2] = {IF_SVE_CG_2A, IF_SVE_CJ_2A}; + const static insFormat formatEncode2AF[2] = {IF_SVE_AE_3A, IF_SVE_BD_3A}; + const static insFormat formatEncode2AG[2] = {IF_SVE_BS_1A, IF_SVE_CZ_4A}; + const static insFormat formatEncode2AH[2] = {IF_SVE_BQ_2A, IF_SVE_BQ_2B}; + const static insFormat formatEncode2AI[2] = {IF_SVE_AM_2A, IF_SVE_EU_3A}; + const static insFormat formatEncode2AJ[2] = {IF_SVE_HI_3A, IF_SVE_HT_4A}; + const static insFormat formatEncode2AK[2] = {IF_SVE_BZ_3A, IF_SVE_BZ_3A_A}; + const static insFormat formatEncode2AL[2] = {IF_SVE_GG_3A, IF_SVE_GG_3B}; + const static insFormat formatEncode2AM[2] = {IF_SVE_HL_3A, IF_SVE_HM_2A}; + const static insFormat formatEncode2AN[2] = {IF_SVE_EI_3A, IF_SVE_EZ_3A}; + const static insFormat formatEncode2AO[2] = {IF_SVE_GT_4A, IF_SVE_GV_3A}; + const static insFormat formatEncode2AP[2] = {IF_SVE_GY_3B, IF_SVE_HA_3A}; + const static insFormat formatEncode2AQ[2] = {IF_SVE_GO_3A, IF_SVE_HC_3A}; + const static insFormat formatEncode2AR[2] = {IF_SVE_AP_3A, IF_SVE_CZ_4A}; + const static insFormat formatEncode2AT[2] = {IF_SVE_AB_3A, IF_SVE_EC_1A}; + const static insFormat formatEncode2AU[2] = {IF_SVE_AH_3A, IF_SVE_BI_2A}; + const static insFormat formatEncode2AV[2] = {IF_SVE_BM_1A, IF_SVE_BN_1A}; + const static insFormat formatEncode2AW[2] = {IF_SVE_BO_1A, IF_SVE_BP_1A}; + const static insFormat formatEncode2AX[2] = {IF_SVE_CC_2A, IF_SVE_CD_2A}; + const static insFormat formatEncode2AY[2] = {IF_SVE_CR_3A, IF_SVE_CS_3A}; + const static insFormat formatEncode2AZ[2] = {IF_SVE_CV_3A, IF_SVE_CV_3B}; + const static insFormat formatEncode2BA[2] = {IF_SVE_CW_4A, IF_SVE_CZ_4A}; + const static insFormat formatEncode2BB[2] = {IF_SVE_CZ_4A, IF_SVE_CZ_4A_A}; + const static insFormat formatEncode2BC[2] = {IF_SVE_DE_1A, IF_SVE_DZ_1A}; + const static insFormat formatEncode2BD[2] = {IF_SVE_DG_2A, IF_SVE_DH_1A}; + const static insFormat formatEncode2BE[2] = {IF_SVE_DK_3A, IF_SVE_DL_2A}; + const static insFormat formatEncode2BF[2] = {IF_SVE_DM_2A, IF_SVE_DN_2A}; + const static insFormat formatEncode2BG[2] = {IF_SVE_DO_2A, IF_SVE_DP_2A}; + const static insFormat formatEncode2BH[2] = {IF_SVE_DW_2A, IF_SVE_DW_2B}; + const static insFormat formatEncode2BI[2] = {IF_SVE_FN_3A, IF_SVE_FN_3B}; + const static insFormat formatEncode2BJ[2] = {IF_SVE_GQ_3A, IF_SVE_HG_2A}; + const static insFormat formatEncode2BK[2] = {IF_SVE_GU_3C, IF_SVE_HU_4B}; + const static insFormat formatEncode2BL[2] = {IF_SVE_GZ_3A, IF_SVE_HB_3A}; + const static insFormat formatEncode2BM[2] = {IF_SVE_HK_3B, IF_SVE_HL_3B}; + const static insFormat formatEncode2BN[2] = {IF_SVE_IF_4A, IF_SVE_IF_4A_A}; + const static insFormat formatEncode2BO[2] = {IF_SVE_IO_3A, IF_SVE_IP_4A}; + const static insFormat formatEncode2BP[2] = {IF_SVE_IQ_3A, IF_SVE_IR_4A}; + const static insFormat formatEncode2BQ[2] = {IF_SVE_IS_3A, IF_SVE_IT_4A}; + const static insFormat formatEncode2BR[2] = {IF_SVE_JC_4A, IF_SVE_JO_3A}; + const static insFormat formatEncode2BS[2] = {IF_SVE_JE_3A, IF_SVE_JF_4A}; + + code_t code = BAD_CODE; + insFormat insFmt = emitInsFormat(ins); + bool encoding_found = false; + int index = -1; + + switch (insFmt) + { + case IF_SVE_13A: + for (index = 0; index < 13; index++) + { + if (fmt == formatEncode13A[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_rdffr: - opt = INS_OPTS_SCALABLE_B; - assert(isPredicateRegister(reg)); // DDDD - fmt = IF_SVE_DH_1A; + case IF_SVE_11A: + for (index = 0; index < 11; index++) + { + if (fmt == formatEncode11A[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_pfalse: - opt = INS_OPTS_SCALABLE_B; - assert(isPredicateRegister(reg)); // DDDD - fmt = IF_SVE_DJ_1A; + case IF_SVE_9A: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9A[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_wrffr: - opt = INS_OPTS_SCALABLE_B; - assert(isPredicateRegister(reg)); // NNNN - fmt = IF_SVE_DR_1A; + case IF_SVE_9B: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9B[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_ptrue: - assert(insOptsScalableStandard(opt)); - assert(isHighPredicateRegister(reg)); // DDD - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_DZ_1A; + case IF_SVE_9C: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9C[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_fmov: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EB_1B; - - // FMOV is a pseudo-instruction for DUP, which is aliased by MOV; - // MOV is the preferred disassembly - ins = INS_sve_mov; + case IF_SVE_9D: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9D[index]) + { + encoding_found = true; + break; + } + } break; - - default: - unreached(); + case IF_SVE_9E: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9E[index]) + { + encoding_found = true; + break; + } + } break; - } - - instrDesc* id = emitNewInstrSmall(attr); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - id->idReg1(reg); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add a SVE instruction referencing a register and a constant. - */ - -void emitter::emitInsSve_R_I(instruction ins, - emitAttr attr, - regNumber reg, - ssize_t imm, - insOpts opt, /* = INS_OPTS_NONE */ - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) -{ - emitAttr size = EA_SIZE(attr); - bool canEncode = false; - bool signedImm = false; - bool hasShift = false; - insFormat fmt; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - bitMaskImm bmi; - - case INS_sve_rdvl: - assert(insOptsNone(opt)); - assert(size == EA_8BYTE); - assert(isGeneralRegister(reg)); // ddddd - assert(isValidSimm<6>(imm)); // iiiiii - fmt = IF_SVE_BC_1A; - canEncode = true; + case IF_SVE_9F: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9F[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_smax: - case INS_sve_smin: - signedImm = true; - - FALLTHROUGH; - case INS_sve_umax: - case INS_sve_umin: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (signedImm) + case IF_SVE_8A: + for (index = 0; index < 8; index++) { - assert(isValidSimm<8>(imm)); // iiiiiiii + if (fmt == formatEncode8A[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_8B: + for (index = 0; index < 8; index++) { - assert(isValidUimm<8>(imm)); // iiiiiiii + if (fmt == formatEncode8B[index]) + { + encoding_found = true; + break; + } } - - fmt = IF_SVE_ED_1A; - canEncode = true; break; - - case INS_sve_mul: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - assert(isValidSimm<8>(imm)); // iiiiiiii - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EE_1A; - canEncode = true; + case IF_SVE_8C: + for (index = 0; index < 8; index++) + { + if (fmt == formatEncode8C[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_mov: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - if (sopt == INS_SCALABLE_OPTS_IMM_BITMASK) + case IF_SVE_7A: + for (index = 0; index < 7; index++) { - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - - if (!useMovDisasmForBitMask(imm)) + if (fmt == formatEncode7A[index]) { - ins = INS_sve_dupm; + encoding_found = true; + break; } - - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BT_1A; } - else + break; + case IF_SVE_6A: + for (index = 0; index < 6; index++) { - assert(insScalableOptsNone(sopt)); - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (!isValidSimm<8>(imm)) + if (fmt == formatEncode6A[index]) { - // Size specifier must be able to fit a left-shifted immediate - assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii - assert(insOptsScalableAtLeastHalf(opt)); - hasShift = true; - imm >>= 8; + encoding_found = true; + break; } - - fmt = IF_SVE_EB_1A; - canEncode = true; } break; - - case INS_sve_dup: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (!isValidSimm<8>(imm)) + case IF_SVE_6B: + for (index = 0; index < 6; index++) { - // Size specifier must be able to fit a left-shifted immediate - assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii - assert(insOptsScalableAtLeastHalf(opt)); - hasShift = true; - imm >>= 8; + if (fmt == formatEncode6B[index]) + { + encoding_found = true; + break; + } } - - fmt = IF_SVE_EB_1A; - canEncode = true; - - // MOV is an alias for DUP, and is always the preferred disassembly. - ins = INS_sve_mov; break; - - case INS_sve_add: - case INS_sve_sub: - case INS_sve_sqadd: - case INS_sve_sqsub: - case INS_sve_uqadd: - case INS_sve_uqsub: - case INS_sve_subr: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - if (!isValidUimm<8>(imm)) + case IF_SVE_6C: + for (index = 0; index < 6; index++) { - // Size specifier must be able to fit left-shifted immediate - assert((isValidUimm_MultipleOf<8, 256>(imm))); // iiiiiiii - assert(insOptsScalableAtLeastHalf(opt)); - hasShift = true; - imm >>= 8; + if (fmt == formatEncode6C[index]) + { + encoding_found = true; + break; + } } - - fmt = IF_SVE_EC_1A; - canEncode = true; - break; - - case INS_sve_and: - case INS_sve_orr: - case INS_sve_eor: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BS_1A; - break; - - case INS_sve_bic: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - // AND is an alias for BIC, and is always the preferred disassembly. - ins = INS_sve_and; - imm = -imm - 1; - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BS_1A; break; - - case INS_sve_eon: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - // EOR is an alias for EON, and is always the preferred disassembly. - ins = INS_sve_eor; - imm = -imm - 1; - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BS_1A; + case IF_SVE_6D: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6D[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_orn: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - // ORR is an alias for ORN, and is always the preferred disassembly. - ins = INS_sve_orr; - imm = -imm - 1; - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BS_1A; + case IF_SVE_6E: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6E[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_dupm: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - fmt = IF_SVE_BT_1A; - - if (useMovDisasmForBitMask(imm)) + case IF_SVE_6F: + for (index = 0; index < 6; index++) { - ins = INS_sve_mov; + if (fmt == formatEncode6F[index]) + { + encoding_found = true; + break; + } } - - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); break; - - default: - unreached(); + case IF_SVE_6G: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6G[index]) + { + encoding_found = true; + break; + } + } break; - } - - assert(canEncode); - - // For encodings with shifted immediates, we need a way to determine if the immediate has been shifted or not. - // We could just leave the immediate in its unshifted form, and call emitNewInstrSC, - // but that would allocate unnecessarily large descriptors. Therefore: - // - For encodings without any shifting, just call emitNewInstrSC. - // - For unshifted immediates, call emitNewInstrSC. - // If it allocates a small descriptor, idHasShift() will always return false. - // Else, idHasShift still returns false, as we set the dedicated bit in large descriptors to false. - // - For immediates that need a shift, call emitNewInstrCns so a normal or large descriptor is used. - // idHasShift will always check the dedicated bit, as it is always available. We set this bit to true below. - instrDesc* id = !hasShift ? emitNewInstrSC(attr, imm) : emitNewInstrCns(attr, imm); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg); - - id->idHasShift(hasShift); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add a SVE instruction referencing a register and a floating point constant. - */ - -void emitter::emitInsSve_R_F( - instruction ins, emitAttr attr, regNumber reg, double immDbl, insOpts opt /* = INS_OPTS_NONE */) -{ - ssize_t imm = 0; - bool canEncode = false; - insFormat fmt; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - floatImm8 fpi; - - case INS_sve_fmov: - case INS_sve_fdup: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg)); // ddddd - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - fpi.immFPIVal = 0; - canEncode = canEncodeFloatImm8(immDbl, &fpi); - imm = fpi.immFPIVal; - fmt = IF_SVE_EA_1A; - - // FMOV is an alias for FDUP, and is always the preferred disassembly. - ins = INS_sve_fmov; + case IF_SVE_5A: + for (index = 0; index < 5; index++) + { + if (fmt == formatEncode5A[index]) + { + encoding_found = true; + break; + } + } break; - - default: - unreached(); + case IF_SVE_5B: + for (index = 0; index < 5; index++) + { + if (fmt == formatEncode5B[index]) + { + encoding_found = true; + break; + } + } break; - } - - assert(canEncode); - - instrDesc* id = emitNewInstrSC(attr, imm); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add a SVE instruction referencing two registers - */ - -void emitter::emitInsSve_R_R(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) -{ - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_sve_pmov: - if (opt != INS_OPTS_SCALABLE_B) + case IF_SVE_5C: + for (index = 0; index < 5; index++) { - assert(insOptsScalableStandard(opt)); - return emitInsSve_R_R_I(INS_sve_pmov, attr, reg1, reg2, 0, opt, sopt); + if (fmt == formatEncode5C[index]) + { + encoding_found = true; + break; + } } - if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) + break; + case IF_SVE_5D: + for (index = 0; index < 5; index++) { - assert(isPredicateRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_SVE_CE_2A; + if (fmt == formatEncode5D[index]) + { + encoding_found = true; + break; + } } - else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) + break; + case IF_SVE_5E: + for (index = 0; index < 5; index++) { - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - fmt = IF_SVE_CF_2A; + if (fmt == formatEncode5E[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_4A: + for (index = 0; index < 4; index++) { - assert(!"invalid instruction"); + if (fmt == formatEncode4A[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_movs: - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // dddd - assert(isPredicateRegister(reg2)); // nnnn - fmt = IF_SVE_CZ_4A_A; + case IF_SVE_4B: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4B[index]) + { + encoding_found = true; + break; + } + } break; - } - - case INS_sve_mov: - { - if (isGeneralRegisterOrSP(reg2)) + case IF_SVE_4E: + for (index = 0; index < 4; index++) { - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); -#ifdef DEBUG - if (opt == INS_OPTS_SCALABLE_D) + if (fmt == formatEncode4E[index]) { - assert(size == EA_8BYTE); + encoding_found = true; + break; } - else + } + break; + case IF_SVE_4F: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4F[index]) { - assert(size == EA_4BYTE); + encoding_found = true; + break; } -#endif // DEBUG - reg2 = encodingSPtoZR(reg2); - fmt = IF_SVE_CB_2A; } - else + break; + case IF_SVE_4G: + for (index = 0; index < 4; index++) { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // dddd - assert(isPredicateRegister(reg2)); // nnnn - fmt = IF_SVE_CZ_4A_L; + if (fmt == formatEncode4G[index]) + { + encoding_found = true; + break; + } } break; - } - - case INS_sve_insr: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - if (isVectorRegister(reg2)) + case IF_SVE_4H: + for (index = 0; index < 4; index++) { - fmt = IF_SVE_CC_2A; + if (fmt == formatEncode4H[index]) + { + encoding_found = true; + break; + } } - else if (isGeneralRegisterOrZR(reg2)) + break; + case IF_SVE_4I: + for (index = 0; index < 4; index++) { - fmt = IF_SVE_CD_2A; + if (fmt == formatEncode4I[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_4J: + for (index = 0; index < 4; index++) { - unreached(); + if (fmt == formatEncode4J[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_pfirst: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // gggg - fmt = IF_SVE_DD_2A; + case IF_SVE_4K: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4K[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_pnext: - assert(insOptsScalableStandard(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // VVVV - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_DF_2A; + case IF_SVE_4L: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4L[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_punpkhi: - case INS_sve_punpklo: - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // NNNN - fmt = IF_SVE_CK_2A; + case IF_SVE_3A: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3A[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_rdffr: - case INS_sve_rdffrs: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // gggg - fmt = IF_SVE_DG_2A; + case IF_SVE_3B: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3B[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_rev: - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + case IF_SVE_3C: + for (index = 0; index < 3; index++) { - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_CG_2A; + if (fmt == formatEncode3C[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_3D: + for (index = 0; index < 3; index++) { - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // NNNN - fmt = IF_SVE_CJ_2A; + if (fmt == formatEncode3D[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_ptest: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // gggg - assert(isPredicateRegister(reg2)); // NNNN - fmt = IF_SVE_DI_2A; + case IF_SVE_3E: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3E[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_cntp: - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsWithVectorLength(sopt)); // l - assert(isGeneralRegister(reg1)); // ddddd - assert(isPredicateRegister(reg2)); // NNNN - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_DL_2A; + case IF_SVE_3F: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3F[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_incp: - case INS_sve_decp: - assert(isPredicateRegister(reg2)); // MMMM - - if (isGeneralRegister(reg1)) // ddddd + case IF_SVE_3G: + for (index = 0; index < 3; index++) { - assert(insOptsScalableStandard(opt)); // xx - assert(size == EA_8BYTE); - fmt = IF_SVE_DM_2A; + if (fmt == formatEncode3G[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_3H: + for (index = 0; index < 3; index++) { - assert(insOptsScalableAtLeastHalf(opt)); // xx - assert(isVectorRegister(reg1)); // ddddd - assert(isScalableVectorSize(size)); - fmt = IF_SVE_DN_2A; + if (fmt == formatEncode3H[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_sqincp: - case INS_sve_uqincp: - case INS_sve_sqdecp: - case INS_sve_uqdecp: - assert(isPredicateRegister(reg2)); // MMMM - - if (isGeneralRegister(reg1)) // ddddd + case IF_SVE_3I: + for (index = 0; index < 3; index++) { - assert(insOptsScalableStandard(opt)); // xx - assert(isValidGeneralDatasize(size)); - fmt = IF_SVE_DO_2A; + if (fmt == formatEncode3I[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_3J: + for (index = 0; index < 3; index++) { - assert(insOptsScalableAtLeastHalf(opt)); // xx - assert(isVectorRegister(reg1)); // ddddd - assert(isScalableVectorSize(size)); - fmt = IF_SVE_DP_2A; + if (fmt == formatEncode3J[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_ctermeq: - case INS_sve_ctermne: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg1)); // nnnnn - assert(isGeneralRegister(reg2)); // mmmmm - assert(isValidGeneralDatasize(size)); // x - fmt = IF_SVE_DS_2A; + case IF_SVE_3K: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3K[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_sqcvtn: - case INS_sve_uqcvtn: - case INS_sve_sqcvtun: - assert(insOptsNone(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowVectorRegister(reg2)); // nnnn - fmt = IF_SVE_FZ_2A; + case IF_SVE_3L: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3L[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_fcvtn: - case INS_sve_bfcvtn: - case INS_sve_fcvtnt: - case INS_sve_fcvtnb: - unreached(); // TODO-SVE: Not yet supported. - assert(insOptsNone(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowVectorRegister(reg2)); // nnnn - fmt = IF_SVE_HG_2A; + case IF_SVE_3M: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3M[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_sqxtnb: - case INS_sve_sqxtnt: - case INS_sve_uqxtnb: - case INS_sve_uqxtnt: - case INS_sve_sqxtunb: - case INS_sve_sqxtunt: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(optGetSveElemsize(opt) != EA_8BYTE); - assert(isValidVectorElemsize(optGetSveElemsize(opt))); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_GD_2A; + case IF_SVE_3N: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3N[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_aese: - case INS_sve_aesd: - case INS_sve_sm4e: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); -#ifdef DEBUG - if (opt == INS_OPTS_SCALABLE_S) + case IF_SVE_3O: + for (index = 0; index < 3; index++) { - assert(ins == INS_sve_sm4e); + if (fmt == formatEncode3O[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_3P: + for (index = 0; index < 3; index++) { - assert(opt == INS_OPTS_SCALABLE_B); + if (fmt == formatEncode3P[index]) + { + encoding_found = true; + break; + } } -#endif // DEBUG - fmt = IF_SVE_GK_2A; - break; - - case INS_sve_frecpe: - case INS_sve_frsqrte: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_HF_2A; - break; - - case INS_sve_sunpkhi: - case INS_sve_sunpklo: - case INS_sve_uunpkhi: - case INS_sve_uunpklo: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_CH_2A; break; - - case INS_sve_fexpa: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_BJ_2A; + case IF_SVE_3Q: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3Q[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_dup: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrSP(reg2)); -#ifdef DEBUG - if (opt == INS_OPTS_SCALABLE_D) + case IF_SVE_3R: + for (index = 0; index < 3; index++) { - assert(size == EA_8BYTE); + if (fmt == formatEncode3R[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_3S: + for (index = 0; index < 3; index++) { - assert(size == EA_4BYTE); + if (fmt == formatEncode3S[index]) + { + encoding_found = true; + break; + } } -#endif // DEBUG - reg2 = encodingSPtoZR(reg2); - fmt = IF_SVE_CB_2A; - - // DUP is an alias for MOV; - // MOV is the preferred disassembly - ins = INS_sve_mov; break; - - case INS_sve_bf1cvt: - case INS_sve_bf1cvtlt: - case INS_sve_bf2cvt: - case INS_sve_bf2cvtlt: - case INS_sve_f1cvt: - case INS_sve_f1cvtlt: - case INS_sve_f2cvt: - case INS_sve_f2cvtlt: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_HH_2A; - unreached(); // not supported yet + case IF_SVE_3T: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3T[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_movprfx: - assert(insScalableOptsNone(sopt)); - assert(insOptsNone(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_BI_2A; + case IF_SVE_3U: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3U[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_fmov: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isPredicateRegister(reg2)); // gggg - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_BV_2B; - - // CPY is an alias for FMOV, and MOV is an alias for CPY. - // Thus, MOV is the preferred disassembly. - ins = INS_sve_mov; + case IF_SVE_3V: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3V[index]) + { + encoding_found = true; + break; + } + } break; - - default: - unreached(); + case IF_SVE_2AA: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AA[index]) + { + encoding_found = true; + break; + } + } break; - } - - assert(fmt != IF_NONE); - - instrDesc* id; - - if (insScalableOptsWithVectorLength(sopt)) - { - id = emitNewInstr(attr); - id->idVectorLength4x(sopt == INS_SCALABLE_OPTS_VL_4X); - } - else - { - id = emitNewInstrSmall(attr); - } - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg1); - id->idReg2(reg2); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add a SVE instruction referencing a register and two constants. - */ - -void emitter::emitInsSve_R_I_I( - instruction ins, emitAttr attr, regNumber reg, ssize_t imm1, ssize_t imm2, insOpts opt /* = INS_OPTS_NONE */) -{ - insFormat fmt; - ssize_t immOut; - - if (ins == INS_sve_index) - { - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - assert(isValidSimm<5>(imm1)); // iiiii - assert(isValidSimm<5>(imm2)); // iiiii - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - immOut = insEncodeTwoSimm5(imm1, imm2); - fmt = IF_SVE_AX_1A; - } - else - { - unreached(); - } - - instrDesc* id = emitNewInstrSC(attr, immOut); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - - id->idReg1(reg); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add a SVE instruction referencing two registers and a constant. - */ - -void emitter::emitInsSve_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) -{ - emitAttr size = EA_SIZE(attr); - bool hasShift = false; - insFormat fmt; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - bool isRightShift; - - case INS_sve_asr: - case INS_sve_lsl: - case INS_sve_lsr: - case INS_sve_srshr: - case INS_sve_sqshl: - case INS_sve_urshr: - case INS_sve_sqshlu: - case INS_sve_uqshl: - case INS_sve_asrd: - isRightShift = emitInsIsVectorRightShift(ins); - assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); - assert(insOptsScalableStandard(opt)); - assert(isScalableVectorSize(size)); - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + case IF_SVE_2AB: + for (index = 0; index < 2; index++) { - assert((ins == INS_sve_asr) || (ins == INS_sve_lsl) || (ins == INS_sve_lsr)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_SVE_BF_2A; + if (fmt == formatEncode2AB[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_2AC: + for (index = 0; index < 2; index++) { - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_AM_2A; + if (fmt == formatEncode2AC[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_xar: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx xx - - switch (opt) + case IF_SVE_2AD: + for (index = 0; index < 2; index++) { - case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); // iii - break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); // xiii + if (fmt == formatEncode2AD[index]) + { + encoding_found = true; break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); // xxiii + } + } + break; + case IF_SVE_2AE: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AE[index]) + { + encoding_found = true; break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimmFrom1<6>(imm)); // x xxiii + } + } + break; + case IF_SVE_2AF: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AF[index]) + { + encoding_found = true; break; - - default: - unreached(); + } + } + break; + case IF_SVE_2AG: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AG[index]) + { + encoding_found = true; break; + } } - - fmt = IF_SVE_AW_2A; break; - - case INS_sve_index: - assert(insOptsScalable(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isValidSimm<5>(imm)); // iiiii - assert(isIntegerRegister(reg2)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (sopt == INS_SCALABLE_OPTS_IMM_FIRST) + case IF_SVE_2AH: + for (index = 0; index < 2; index++) { - fmt = IF_SVE_AY_2A; + if (fmt == formatEncode2AH[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_2AI: + for (index = 0; index < 2; index++) { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AZ_2A; + if (fmt == formatEncode2AI[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_addvl: - case INS_sve_addpl: - assert(insOptsNone(opt)); - assert(size == EA_8BYTE); - assert(isGeneralRegisterOrSP(reg1)); // ddddd - assert(isGeneralRegisterOrSP(reg2)); // nnnnn - assert(isValidSimm<6>(imm)); // iiiiii - reg1 = encodingSPtoZR(reg1); - reg2 = encodingSPtoZR(reg2); - fmt = IF_SVE_BB_2A; + case IF_SVE_2AJ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AJ[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_mov: - if (sopt == INS_SCALABLE_OPTS_BROADCAST) + case IF_SVE_2AK: + for (index = 0; index < 2; index++) { - return emitInsSve_R_R_I(INS_sve_dup, attr, reg1, reg2, imm, opt, sopt); + if (fmt == formatEncode2AK[index]) + { + encoding_found = true; + break; + } } - FALLTHROUGH; - case INS_sve_cpy: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // DDDDD - assert(isPredicateRegister(reg2)); // GGGG - - if (!isValidSimm<8>(imm)) + break; + case IF_SVE_2AL: + for (index = 0; index < 2; index++) { - // Size specifier must be able to fit a left-shifted immediate - assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii - assert(insOptsScalableAtLeastHalf(opt)); - hasShift = true; - imm >>= 8; + if (fmt == formatEncode2AL[index]) + { + encoding_found = true; + break; + } } - - if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) + break; + case IF_SVE_2AM: + for (index = 0; index < 2; index++) { - fmt = IF_SVE_BV_2A_J; + if (fmt == formatEncode2AM[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_2AN: + for (index = 0; index < 2; index++) { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BV_2A; + if (fmt == formatEncode2AN[index]) + { + encoding_found = true; + break; + } } - - // MOV is an alias for CPY, and is always the preferred disassembly. - ins = INS_sve_mov; break; - - case INS_sve_dup: - assert(insOptsScalable(opt)); - assert(isVectorRegister(reg1)); // DDDDD - assert(isVectorRegister(reg2)); // GGGG - assert(isValidBroadcastImm(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BW_2A; - ins = INS_sve_mov; // Set preferred alias for disassembly + case IF_SVE_2AO: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AO[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_pmov: - if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) + case IF_SVE_2AP: + for (index = 0; index < 2; index++) { - assert(isPredicateRegister(reg1)); - assert(isVectorRegister(reg2)); - switch (opt) + if (fmt == formatEncode2AP[index]) { - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<3>(imm)); - fmt = IF_SVE_CE_2B; - break; - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - fmt = IF_SVE_CE_2D; - break; - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<1>(imm)); - fmt = IF_SVE_CE_2C; - break; - default: - unreached(); + encoding_found = true; + break; } } - else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) + break; + case IF_SVE_2AQ: + for (index = 0; index < 2; index++) { - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - switch (opt) + if (fmt == formatEncode2AQ[index]) { - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<3>(imm)); - fmt = IF_SVE_CF_2B; - break; - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - fmt = IF_SVE_CF_2D; - break; - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<1>(imm)); - fmt = IF_SVE_CF_2C; - break; - default: - unreached(); + encoding_found = true; + break; } } - else + break; + case IF_SVE_2AR: + for (index = 0; index < 2; index++) { - unreached(); + if (fmt == formatEncode2AR[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_sqrshrn: - case INS_sve_sqrshrun: - case INS_sve_uqrshrn: - isRightShift = emitInsIsVectorRightShift(ins); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(opt == INS_OPTS_SCALABLE_H); - assert(isRightShift); // These are always right-shift. - assert(isValidVectorShiftAmount(imm, EA_4BYTE, isRightShift)); - fmt = IF_SVE_GA_2A; + case IF_SVE_2AT: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AT[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_pext: - assert(insOptsScalableStandard(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isHighPredicateRegister(reg2)); // NNN - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (sopt == INS_SCALABLE_OPTS_WITH_PREDICATE_PAIR) + case IF_SVE_2AU: + for (index = 0; index < 2; index++) { - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_DW_2B; + if (fmt == formatEncode2AU[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_2AV: + for (index = 0; index < 2; index++) { - assert(insScalableOptsNone(sopt)); - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_DW_2A; + if (fmt == formatEncode2AV[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_sshllb: - case INS_sve_sshllt: - case INS_sve_ushllb: - case INS_sve_ushllt: - assert(insOptsScalableWide(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // x xx - - switch (opt) + case IF_SVE_2AW: + for (index = 0; index < 2; index++) { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<3>(imm)); // iii + if (fmt == formatEncode2AW[index]) + { + encoding_found = true; break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<4>(imm)); // x iii + } + } + break; + case IF_SVE_2AX: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AX[index]) + { + encoding_found = true; break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<5>(imm)); // xx iii + } + } + break; + case IF_SVE_2AY: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AY[index]) + { + encoding_found = true; break; - - default: - unreached(); + } + } + break; + case IF_SVE_2AZ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AZ[index]) + { + encoding_found = true; break; + } } - - fmt = IF_SVE_FR_2A; break; - - case INS_sve_sqshrunb: - case INS_sve_sqshrunt: - case INS_sve_sqrshrunb: - case INS_sve_sqrshrunt: - case INS_sve_shrnb: - case INS_sve_shrnt: - case INS_sve_rshrnb: - case INS_sve_rshrnt: - case INS_sve_sqshrnb: - case INS_sve_sqshrnt: - case INS_sve_sqrshrnb: - case INS_sve_sqrshrnt: - case INS_sve_uqshrnb: - case INS_sve_uqshrnt: - case INS_sve_uqrshrnb: - case INS_sve_uqrshrnt: - assert(insOptsScalableWide(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // x xx - - switch (opt) + case IF_SVE_2BA: + for (index = 0; index < 2; index++) { - case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); // iii - break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); // x iii + if (fmt == formatEncode2BA[index]) + { + encoding_found = true; break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); // xx iii + } + } + break; + case IF_SVE_2BB: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BB[index]) + { + encoding_found = true; break; - - default: - unreached(); + } + } + break; + case IF_SVE_2BC: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BC[index]) + { + encoding_found = true; break; + } } - - fmt = IF_SVE_GB_2A; break; - - case INS_sve_cadd: - case INS_sve_sqcadd: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - // Convert rot to bitwise representation: 0 if 90, 1 if 270 - imm = emitEncodeRotationImm90_or_270(imm); // r - fmt = IF_SVE_FV_2A; + case IF_SVE_2BD: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BD[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_ftmad: - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isValidUimm<3>(imm)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_HN_2A; + case IF_SVE_2BE: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BE[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_ldr: - assert(insOptsNone(opt)); - assert(isScalableVectorSize(size)); - assert(isGeneralRegister(reg2)); // nnnnn - assert(isValidSimm<9>(imm)); // iii - // iiiiii - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + case IF_SVE_2BF: + for (index = 0; index < 2; index++) { - assert(isVectorRegister(reg1)); - fmt = IF_SVE_IE_2A; + if (fmt == formatEncode2BF[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_2BG: + for (index = 0; index < 2; index++) { - assert(insScalableOptsNone(sopt)); - assert(isPredicateRegister(reg1)); - fmt = IF_SVE_ID_2A; + if (fmt == formatEncode2BG[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_str: - assert(insOptsNone(opt)); - assert(isScalableVectorSize(size)); - assert(isGeneralRegister(reg2)); // nnnnn - assert(isValidSimm<9>(imm)); // iii - // iiiiii - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + case IF_SVE_2BH: + for (index = 0; index < 2; index++) { - assert(isVectorRegister(reg1)); - fmt = IF_SVE_JH_2A; + if (fmt == formatEncode2BH[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_2BI: + for (index = 0; index < 2; index++) { - assert(insScalableOptsNone(sopt)); - assert(isPredicateRegister(reg1)); - fmt = IF_SVE_JG_2A; + if (fmt == formatEncode2BI[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_sli: - case INS_sve_sri: - isRightShift = emitInsIsVectorRightShift(ins); - assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_FT_2A; + case IF_SVE_2BJ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BJ[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_srsra: - case INS_sve_ssra: - case INS_sve_ursra: - case INS_sve_usra: - isRightShift = emitInsIsVectorRightShift(ins); - assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_FU_2A; + case IF_SVE_2BK: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BK[index]) + { + encoding_found = true; + break; + } + } break; - - case INS_sve_ext: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isValidUimm<8>(imm)); // iiiii iii - - if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) + case IF_SVE_2BL: + for (index = 0; index < 2; index++) { - fmt = IF_SVE_BQ_2A; + if (fmt == formatEncode2BL[index]) + { + encoding_found = true; + break; + } } - else + break; + case IF_SVE_2BM: + for (index = 0; index < 2; index++) { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BQ_2B; + if (fmt == formatEncode2BM[index]) + { + encoding_found = true; + break; + } } break; - - case INS_sve_dupq: - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); -#ifdef DEBUG - switch (opt) + case IF_SVE_2BN: + for (index = 0; index < 2; index++) { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); + if (fmt == formatEncode2BN[index]) + { + encoding_found = true; break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); + } + } + break; + case IF_SVE_2BO: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BO[index]) + { + encoding_found = true; break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); + } + } + break; + case IF_SVE_2BP: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BP[index]) + { + encoding_found = true; break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); + } + } + break; + case IF_SVE_2BQ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BQ[index]) + { + encoding_found = true; break; - - default: + } + } + break; + case IF_SVE_2BR: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BR[index]) + { + encoding_found = true; break; + } } -#endif // DEBUG - fmt = IF_SVE_BX_2A; break; - - case INS_sve_extq: - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - assert(isValidUimm<4>(imm)); - fmt = IF_SVE_BY_2A; + case IF_SVE_2BS: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BS[index]) + { + encoding_found = true; + break; + } + } break; - default: - unreached(); + if (fmt == insFmt) + { + encoding_found = true; + index = 0; + } + else + { + encoding_found = false; + } break; } - // For encodings with shifted immediates, we need a way to determine if the immediate has been shifted or not. - // We could just leave the immediate in its unshifted form, and call emitNewInstrSC, - // but that would allocate unnecessarily large descriptors. Therefore: - // - For encodings without any shifting, just call emitNewInstrSC. - // - For unshifted immediates, call emitNewInstrSC. - // If it allocates a small descriptor, idHasShift() will always return false. - // Else, idHasShift still returns false, as we set the dedicated bit in large descriptors to false. - // - For immediates that need a shift, call emitNewInstrCns so a normal or large descriptor is used. - // idHasShift will always check the dedicated bit, as it is always available. We set this bit to true below. - instrDesc* id = !hasShift ? emitNewInstrSC(attr, imm) : emitNewInstrCns(attr, imm); + assert(encoding_found); + const unsigned sve_ins_offset = ((unsigned)ins - INS_sve_invalid); - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); + switch (index) + { + case 0: + assert(sve_ins_offset < ArrLen(insCodes1)); + code = insCodes1[sve_ins_offset]; + break; + case 1: + assert(sve_ins_offset < ArrLen(insCodes2)); + code = insCodes2[sve_ins_offset]; + break; + case 2: + assert(sve_ins_offset < ArrLen(insCodes3)); + code = insCodes3[sve_ins_offset]; + break; + case 3: + assert(sve_ins_offset < ArrLen(insCodes4)); + code = insCodes4[sve_ins_offset]; + break; + case 4: + assert(sve_ins_offset < ArrLen(insCodes5)); + code = insCodes5[sve_ins_offset]; + break; + case 5: + assert(sve_ins_offset < ArrLen(insCodes6)); + code = insCodes6[sve_ins_offset]; + break; + case 6: + assert(sve_ins_offset < ArrLen(insCodes7)); + code = insCodes7[sve_ins_offset]; + break; + case 7: + assert(sve_ins_offset < ArrLen(insCodes8)); + code = insCodes8[sve_ins_offset]; + break; + case 8: + assert(sve_ins_offset < ArrLen(insCodes9)); + code = insCodes9[sve_ins_offset]; + break; + case 9: + assert(sve_ins_offset < ArrLen(insCodes10)); + code = insCodes10[sve_ins_offset]; + break; + case 10: + assert(sve_ins_offset < ArrLen(insCodes11)); + code = insCodes11[sve_ins_offset]; + break; + case 11: + assert(sve_ins_offset < ArrLen(insCodes12)); + code = insCodes12[sve_ins_offset]; + break; + case 12: + assert(sve_ins_offset < ArrLen(insCodes13)); + code = insCodes13[sve_ins_offset]; + break; + } - id->idReg1(reg1); - id->idReg2(reg2); + assert((code != BAD_CODE)); - id->idHasShift(hasShift); + return code; +} + +/***************************************************************************** + * + * Add a SVE instruction with a single immediate value. + */ + +void emitter::emitInsSve_I(instruction ins, emitAttr attr, ssize_t imm) +{ + insFormat fmt; + + /* Figure out the encoding format of the instruction */ + if (ins == INS_sve_setffr) + { + fmt = IF_SVE_DQ_0A; + attr = EA_PTRSIZE; + imm = 0; + } + else + { + unreached(); + } + + instrDesc* id = emitNewInstrSC(attr, imm); + + id->idIns(ins); + id->idInsFmt(fmt); dispIns(id); appendToCurIG(id); @@ -1343,67 +1530,71 @@ void emitter::emitInsSve_R_R_I(instruction ins, /***************************************************************************** * - * Add a SVE instruction referencing two registers and a floating point constant. + * Add a SVE instruction referencing a single register. */ -void emitter::emitInsSve_R_R_F( - instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, double immDbl, insOpts opt /* = INS_OPTS_NONE */) +void emitter::emitInsSve_R(instruction ins, emitAttr attr, regNumber reg, insOpts opt /* = INS_OPTS_NONE */) { - ssize_t imm = 0; - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; + insFormat fmt; /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_sve_fmul: - case INS_sve_fmaxnm: - case INS_sve_fadd: - case INS_sve_fmax: - case INS_sve_fminnm: - case INS_sve_fsub: - case INS_sve_fmin: - case INS_sve_fsubr: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isScalableVectorSize(size)); - imm = emitEncodeSmallFloatImm(immDbl, ins); - fmt = IF_SVE_HM_2A; + case INS_sve_aesmc: + case INS_sve_aesimc: + opt = INS_OPTS_SCALABLE_B; + assert(isVectorRegister(reg)); // ddddd + assert(isScalableVectorSize(attr)); + fmt = IF_SVE_GL_1A; + break; + + case INS_sve_rdffr: + opt = INS_OPTS_SCALABLE_B; + assert(isPredicateRegister(reg)); // DDDD + fmt = IF_SVE_DH_1A; + break; + + case INS_sve_pfalse: + opt = INS_OPTS_SCALABLE_B; + assert(isPredicateRegister(reg)); // DDDD + fmt = IF_SVE_DJ_1A; + break; + + case INS_sve_wrffr: + opt = INS_OPTS_SCALABLE_B; + assert(isPredicateRegister(reg)); // NNNN + fmt = IF_SVE_DR_1A; + break; + + case INS_sve_ptrue: + assert(insOptsScalableStandard(opt)); + assert(isHighPredicateRegister(reg)); // DDD + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_DZ_1A; break; case INS_sve_fmov: - case INS_sve_fcpy: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isPredicateRegister(reg2)); // gggg + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg)); // ddddd assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - floatImm8 fpi; - fpi.immFPIVal = 0; - canEncodeFloatImm8(immDbl, &fpi); - imm = fpi.immFPIVal; - fmt = IF_SVE_BU_2A; + fmt = IF_SVE_EB_1B; - // FMOV is an alias for FCPY, and is always the preferred disassembly. - ins = INS_sve_fmov; + // FMOV is a pseudo-instruction for DUP, which is aliased by MOV; + // MOV is the preferred disassembly + ins = INS_sve_mov; break; default: unreached(); break; + } - } // end switch (ins) - - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrSC(attr, imm); + instrDesc* id = emitNewInstrSmall(attr); id->idIns(ins); id->idInsFmt(fmt); id->idInsOpt(opt); - - id->idReg1(reg1); - id->idReg2(reg2); + id->idReg1(reg); dispIns(id); appendToCurIG(id); @@ -1411,1495 +1602,5587 @@ void emitter::emitInsSve_R_R_F( /***************************************************************************** * - * Add a SVE instruction referencing three registers. + * Add a SVE instruction referencing a register and a constant. */ -void emitter::emitInsSve_R_R_R(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +void emitter::emitInsSve_R_I(instruction ins, + emitAttr attr, + regNumber reg, + ssize_t imm, + insOpts opt, /* = INS_OPTS_NONE */ + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) { - emitAttr size = EA_SIZE(attr); - bool pmerge = false; - bool vectorLength4x = false; - insFormat fmt = IF_NONE; + emitAttr size = EA_SIZE(attr); + bool canEncode = false; + bool signedImm = false; + bool hasShift = false; + insFormat fmt; /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_sve_and: - case INS_sve_bic: - case INS_sve_eor: - case INS_sve_orr: + bitMaskImm bmi; + + case INS_sve_rdvl: + assert(insOptsNone(opt)); + assert(size == EA_8BYTE); + assert(isGeneralRegister(reg)); // ddddd + assert(isValidSimm<6>(imm)); // iiiiii + fmt = IF_SVE_BC_1A; + canEncode = true; + break; + + case INS_sve_smax: + case INS_sve_smin: + signedImm = true; + + FALLTHROUGH; + case INS_sve_umax: + case INS_sve_umin: assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // mmmmm - assert(isVectorRegister(reg3)); // ddddd + assert(isVectorRegister(reg)); // ddddd + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + if (signedImm) { - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg2)); // nnnnn - fmt = IF_SVE_AU_3A; + assert(isValidSimm<8>(imm)); // iiiiiiii } else { - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_AA_3A; + assert(isValidUimm<8>(imm)); // iiiiiiii } + + fmt = IF_SVE_ED_1A; + canEncode = true; break; - case INS_sve_add: - case INS_sve_sub: - case INS_sve_subr: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg3)); + case INS_sve_mul: assert(insOptsScalableStandard(opt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg2)); - assert(ins != INS_sve_subr); - fmt = IF_SVE_AT_3A; - } - else - { - assert(isLowPredicateRegister(reg2)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AB_3A; - } + assert(isVectorRegister(reg)); // ddddd + assert(isValidSimm<8>(imm)); // iiiiiiii + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_EE_1A; + canEncode = true; break; - case INS_sve_addpt: - case INS_sve_subpt: - unreached(); // TODO-SVE: Not yet supported. - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg3)); // mmmmm + case INS_sve_mov: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg)); // ddddd - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + if (sopt == INS_SCALABLE_OPTS_IMM_BITMASK) { - assert(isVectorRegister(reg2)); // nnnnn - fmt = IF_SVE_AT_3B; + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); + + if (!useMovDisasmForBitMask(imm)) + { + ins = INS_sve_dupm; + } + + imm = bmi.immNRS; // iiiiiiiiiiiii + assert(isValidImmNRS(imm, optGetSveElemsize(opt))); + fmt = IF_SVE_BT_1A; } else { assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_AB_3B; + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + if (!isValidSimm<8>(imm)) + { + // Size specifier must be able to fit a left-shifted immediate + assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii + assert(insOptsScalableAtLeastHalf(opt)); + hasShift = true; + imm >>= 8; + } + + fmt = IF_SVE_EB_1A; + canEncode = true; } break; - case INS_sve_sdiv: - case INS_sve_sdivr: - case INS_sve_udiv: - case INS_sve_udivr: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableWords(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AC_3A; - break; - - case INS_sve_sabd: - case INS_sve_smax: - case INS_sve_smin: - case INS_sve_uabd: - case INS_sve_umax: - case INS_sve_umin: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); + case INS_sve_dup: assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AD_3A; - break; + assert(isVectorRegister(reg)); // ddddd + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - case INS_sve_mul: - case INS_sve_smulh: - case INS_sve_umulh: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg2)); - fmt = IF_SVE_BD_3A; - } - else + if (!isValidSimm<8>(imm)) { - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); - fmt = IF_SVE_AE_3A; + // Size specifier must be able to fit a left-shifted immediate + assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii + assert(insOptsScalableAtLeastHalf(opt)); + hasShift = true; + imm >>= 8; } - break; - - case INS_sve_pmul: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_BD_3B; - break; - case INS_sve_andv: - case INS_sve_eorv: - case INS_sve_orv: - assert(isFloatReg(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AF_3A; - break; + fmt = IF_SVE_EB_1A; + canEncode = true; - case INS_sve_andqv: - case INS_sve_eorqv: - case INS_sve_orqv: - unreached(); // TODO-SVE: Not yet supported. - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AG_3A; + // MOV is an alias for DUP, and is always the preferred disassembly. + ins = INS_sve_mov; break; - case INS_sve_movprfx: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); + case INS_sve_add: + case INS_sve_sub: + case INS_sve_sqadd: + case INS_sve_sqsub: + case INS_sve_uqadd: + case INS_sve_uqsub: + case INS_sve_subr: assert(insOptsScalableStandard(opt)); - if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) + assert(isVectorRegister(reg)); // ddddd + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + if (!isValidUimm<8>(imm)) { - pmerge = true; + // Size specifier must be able to fit left-shifted immediate + assert((isValidUimm_MultipleOf<8, 256>(imm))); // iiiiiiii + assert(insOptsScalableAtLeastHalf(opt)); + hasShift = true; + imm >>= 8; } - fmt = IF_SVE_AH_3A; - break; - case INS_sve_saddv: - case INS_sve_uaddv: - assert(isFloatReg(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableWide(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AI_3A; + fmt = IF_SVE_EC_1A; + canEncode = true; break; - case INS_sve_addqv: - unreached(); // TODO-SVE: Not yet supported. - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); + case INS_sve_and: + case INS_sve_orr: + case INS_sve_eor: assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AJ_3A; + assert(isVectorRegister(reg)); // ddddd + + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); + imm = bmi.immNRS; // iiiiiiiiiiiii + assert(isValidImmNRS(imm, optGetSveElemsize(opt))); + fmt = IF_SVE_BS_1A; break; - case INS_sve_smaxv: - case INS_sve_sminv: - case INS_sve_umaxv: - case INS_sve_uminv: - assert(isFloatReg(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); + case INS_sve_bic: assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AK_3A; + assert(isVectorRegister(reg)); // ddddd + + // AND is an alias for BIC, and is always the preferred disassembly. + ins = INS_sve_and; + imm = -imm - 1; + + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); + imm = bmi.immNRS; // iiiiiiiiiiiii + assert(isValidImmNRS(imm, optGetSveElemsize(opt))); + fmt = IF_SVE_BS_1A; break; - case INS_sve_smaxqv: - case INS_sve_sminqv: - case INS_sve_umaxqv: - case INS_sve_uminqv: - unreached(); // TODO-SVE: Not yet supported. - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); + case INS_sve_eon: assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AL_3A; + assert(isVectorRegister(reg)); // ddddd + + // EOR is an alias for EON, and is always the preferred disassembly. + ins = INS_sve_eor; + imm = -imm - 1; + + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); + imm = bmi.immNRS; // iiiiiiiiiiiii + assert(isValidImmNRS(imm, optGetSveElemsize(opt))); + fmt = IF_SVE_BS_1A; break; - case INS_sve_asrr: - case INS_sve_lslr: - case INS_sve_lsrr: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); + case INS_sve_orn: assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AN_3A; - break; + assert(isVectorRegister(reg)); // ddddd - case INS_sve_asr: - case INS_sve_lsl: - case INS_sve_lsr: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg3)); - if (sopt == INS_SCALABLE_OPTS_WIDE) - { - assert(isLowPredicateRegister(reg2)); - assert(insOptsScalableWide(opt)); - fmt = IF_SVE_AO_3A; - } - else if (sopt == INS_SCALABLE_OPTS_UNPREDICATED_WIDE) - { - assert(isVectorRegister(reg2)); - assert(insOptsScalableWide(opt)); - fmt = IF_SVE_BG_3A; - } - else - { - assert(isLowPredicateRegister(reg2)); - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_AN_3A; - } + // ORR is an alias for ORN, and is always the preferred disassembly. + ins = INS_sve_orr; + imm = -imm - 1; + + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); + imm = bmi.immNRS; // iiiiiiiiiiiii + assert(isValidImmNRS(imm, optGetSveElemsize(opt))); + fmt = IF_SVE_BS_1A; break; - case INS_sve_uzp1: - case INS_sve_trn1: - case INS_sve_zip1: - case INS_sve_uzp2: - case INS_sve_trn2: - case INS_sve_zip2: - assert(insOptsScalable(opt)); + case INS_sve_dupm: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg)); // ddddd - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + bmi.immNRS = 0; + canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); + fmt = IF_SVE_BT_1A; - if (opt == INS_OPTS_SCALABLE_Q) - { - fmt = IF_SVE_BR_3B; - } - else - { - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_BR_3A; - } - } - else + if (useMovDisasmForBitMask(imm)) { - assert(insScalableOptsNone(sopt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // NNNN - assert(isPredicateRegister(reg3)); // MMMM - fmt = IF_SVE_CI_3A; + ins = INS_sve_mov; } + + imm = bmi.immNRS; // iiiiiiiiiiiii + assert(isValidImmNRS(imm, optGetSveElemsize(opt))); break; - case INS_sve_tbl: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + default: + unreached(); + break; + } + + assert(canEncode); + + // For encodings with shifted immediates, we need a way to determine if the immediate has been shifted or not. + // We could just leave the immediate in its unshifted form, and call emitNewInstrSC, + // but that would allocate unnecessarily large descriptors. Therefore: + // - For encodings without any shifting, just call emitNewInstrSC. + // - For unshifted immediates, call emitNewInstrSC. + // If it allocates a small descriptor, idHasShift() will always return false. + // Else, idHasShift still returns false, as we set the dedicated bit in large descriptors to false. + // - For immediates that need a shift, call emitNewInstrCns so a normal or large descriptor is used. + // idHasShift will always check the dedicated bit, as it is always available. We set this bit to true below. + instrDesc* id = !hasShift ? emitNewInstrSC(attr, imm) : emitNewInstrCns(attr, imm); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg); + + id->idHasShift(hasShift); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing a register and a floating point constant. + */ + +void emitter::emitInsSve_R_F( + instruction ins, emitAttr attr, regNumber reg, double immDbl, insOpts opt /* = INS_OPTS_NONE */) +{ + ssize_t imm = 0; + bool canEncode = false; + insFormat fmt; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + floatImm8 fpi; + + case INS_sve_fmov: + case INS_sve_fdup: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg)); // ddddd assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) + fpi.immFPIVal = 0; + canEncode = canEncodeFloatImm8(immDbl, &fpi); + imm = fpi.immFPIVal; + fmt = IF_SVE_EA_1A; + + // FMOV is an alias for FDUP, and is always the preferred disassembly. + ins = INS_sve_fmov; + break; + + default: + unreached(); + break; + } + + assert(canEncode); + + instrDesc* id = emitNewInstrSC(attr, imm); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing two registers + */ + +void emitter::emitInsSve_R_R(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +{ + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_pmov: + if (opt != INS_OPTS_SCALABLE_B) { - fmt = IF_SVE_BZ_3A_A; + assert(insOptsScalableStandard(opt)); + return emitInsSve_R_R_I(INS_sve_pmov, attr, reg1, reg2, 0, opt, sopt); + } + if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) + { + assert(isPredicateRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_SVE_CE_2A; + } + else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) + { + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + fmt = IF_SVE_CF_2A; } else { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BZ_3A; + assert(!"invalid instruction"); } break; - case INS_sve_tbx: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_BZ_3A; + case INS_sve_movs: + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // dddd + assert(isPredicateRegister(reg2)); // nnnn + fmt = IF_SVE_CZ_4A_A; break; + } - case INS_sve_tbxq: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_CA_3A; + case INS_sve_mov: + { + if (isGeneralRegisterOrSP(reg2)) + { + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); +#ifdef DEBUG + if (opt == INS_OPTS_SCALABLE_D) + { + assert(size == EA_8BYTE); + } + else + { + assert(size == EA_4BYTE); + } +#endif // DEBUG + reg2 = encodingSPtoZR(reg2); + fmt = IF_SVE_CB_2A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // dddd + assert(isPredicateRegister(reg2)); // nnnn + fmt = IF_SVE_CZ_4A_L; + } break; + } - case INS_sve_sdot: - case INS_sve_udot: + case INS_sve_insr: + assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_H) + if (isVectorRegister(reg2)) { - fmt = IF_SVE_EF_3A; + fmt = IF_SVE_CC_2A; + } + else if (isGeneralRegisterOrZR(reg2)) + { + fmt = IF_SVE_CD_2A; } else { - fmt = IF_SVE_EH_3A; - assert(insOptsScalableWords(opt)); - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + unreached(); } break; - case INS_sve_usdot: + case INS_sve_pfirst: assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_EI_3A; - break; - - case INS_sve_smlalb: - case INS_sve_smlalt: - case INS_sve_umlalb: - case INS_sve_umlalt: - case INS_sve_smlslb: - case INS_sve_smlslt: - case INS_sve_umlslb: - case INS_sve_umlslt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EL_3A; + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // gggg + fmt = IF_SVE_DD_2A; break; - case INS_sve_sqrdmlah: - case INS_sve_sqrdmlsh: + case INS_sve_pnext: assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // VVVV assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EM_3A; + fmt = IF_SVE_DF_2A; break; - case INS_sve_sqdmlalbt: - case INS_sve_sqdmlslbt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EN_3A; + case INS_sve_punpkhi: + case INS_sve_punpklo: + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // NNNN + fmt = IF_SVE_CK_2A; break; - case INS_sve_sqdmlalb: - case INS_sve_sqdmlalt: - case INS_sve_sqdmlslb: - case INS_sve_sqdmlslt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EO_3A; + case INS_sve_rdffr: + case INS_sve_rdffrs: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // gggg + fmt = IF_SVE_DG_2A; break; - case INS_sve_sclamp: - case INS_sve_uclamp: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EV_3A; - break; - - case INS_sve_zipq1: - case INS_sve_zipq2: - case INS_sve_uzpq1: - case INS_sve_uzpq2: - case INS_sve_tblq: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EX_3A; - break; - - case INS_sve_saddlb: - case INS_sve_saddlt: - case INS_sve_uaddlb: - case INS_sve_uaddlt: - case INS_sve_ssublb: - case INS_sve_ssublt: - case INS_sve_usublb: - case INS_sve_usublt: - case INS_sve_sabdlb: - case INS_sve_sabdlt: - case INS_sve_uabdlb: - case INS_sve_uabdlt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FL_3A; + case INS_sve_rev: + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_CG_2A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // NNNN + fmt = IF_SVE_CJ_2A; + } break; - case INS_sve_saddwb: - case INS_sve_saddwt: - case INS_sve_uaddwb: - case INS_sve_uaddwt: - case INS_sve_ssubwb: - case INS_sve_ssubwt: - case INS_sve_usubwb: - case INS_sve_usubwt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FM_3A; + case INS_sve_ptest: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // gggg + assert(isPredicateRegister(reg2)); // NNNN + fmt = IF_SVE_DI_2A; break; - case INS_sve_smullb: - case INS_sve_smullt: - case INS_sve_umullb: - case INS_sve_umullt: - case INS_sve_sqdmullb: - case INS_sve_sqdmullt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + case INS_sve_cntp: + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsWithVectorLength(sopt)); // l + assert(isGeneralRegister(reg1)); // ddddd + assert(isPredicateRegister(reg2)); // NNNN assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FN_3A; + fmt = IF_SVE_DL_2A; break; - case INS_sve_pmullb: - case INS_sve_pmullt: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + case INS_sve_incp: + case INS_sve_decp: + assert(isPredicateRegister(reg2)); // MMMM - if (opt == INS_OPTS_SCALABLE_Q) + if (isGeneralRegister(reg1)) // ddddd { - fmt = IF_SVE_FN_3B; + assert(insOptsScalableStandard(opt)); // xx + assert(size == EA_8BYTE); + fmt = IF_SVE_DM_2A; } else { - assert((opt == INS_OPTS_SCALABLE_H) || (opt == INS_OPTS_SCALABLE_D)); - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FN_3A; + assert(insOptsScalableAtLeastHalf(opt)); // xx + assert(isVectorRegister(reg1)); // ddddd + assert(isScalableVectorSize(size)); + fmt = IF_SVE_DN_2A; } break; - case INS_sve_smmla: - case INS_sve_usmmla: - case INS_sve_ummla: - assert(opt == INS_OPTS_SCALABLE_S); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_FO_3A; - break; - - case INS_sve_rax1: - case INS_sve_sm4ekey: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + case INS_sve_sqincp: + case INS_sve_uqincp: + case INS_sve_sqdecp: + case INS_sve_uqdecp: + assert(isPredicateRegister(reg2)); // MMMM - if (ins == INS_sve_rax1) + if (isGeneralRegister(reg1)) // ddddd { - assert(opt == INS_OPTS_SCALABLE_D); + assert(insOptsScalableStandard(opt)); // xx + assert(isValidGeneralDatasize(size)); + fmt = IF_SVE_DO_2A; } else { - assert(opt == INS_OPTS_SCALABLE_S); + assert(insOptsScalableAtLeastHalf(opt)); // xx + assert(isVectorRegister(reg1)); // ddddd + assert(isScalableVectorSize(size)); + fmt = IF_SVE_DP_2A; } + break; - fmt = IF_SVE_GJ_3A; + case INS_sve_ctermeq: + case INS_sve_ctermne: + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg1)); // nnnnn + assert(isGeneralRegister(reg2)); // mmmmm + assert(isValidGeneralDatasize(size)); // x + fmt = IF_SVE_DS_2A; break; - case INS_sve_fmlalb: - case INS_sve_fmlalt: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + case INS_sve_sqcvtn: + case INS_sve_uqcvtn: + case INS_sve_sqcvtun: + assert(insOptsNone(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowVectorRegister(reg2)); // nnnn + fmt = IF_SVE_FZ_2A; + break; - if (opt == INS_OPTS_SCALABLE_B) + case INS_sve_fcvtn: + case INS_sve_bfcvtn: + case INS_sve_fcvtnt: + case INS_sve_fcvtnb: + unreached(); // TODO-SVE: Not yet supported. + assert(insOptsNone(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowVectorRegister(reg2)); // nnnn + fmt = IF_SVE_HG_2A; + break; + + case INS_sve_sqxtnb: + case INS_sve_sqxtnt: + case INS_sve_uqxtnb: + case INS_sve_uqxtnt: + case INS_sve_sqxtunb: + case INS_sve_sqxtunt: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(optGetSveElemsize(opt) != EA_8BYTE); + assert(isValidVectorElemsize(optGetSveElemsize(opt))); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_GD_2A; + break; + + case INS_sve_aese: + case INS_sve_aesd: + case INS_sve_sm4e: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); +#ifdef DEBUG + if (opt == INS_OPTS_SCALABLE_S) { - unreached(); // TODO-SVE: Not yet supported. - fmt = IF_SVE_GN_3A; + assert(ins == INS_sve_sm4e); } else { - assert(opt == INS_OPTS_SCALABLE_H); - fmt = IF_SVE_HB_3A; + assert(opt == INS_OPTS_SCALABLE_B); } +#endif // DEBUG + fmt = IF_SVE_GK_2A; break; - case INS_sve_fmlslb: - case INS_sve_fmlslt: - case INS_sve_bfmlalb: - case INS_sve_bfmlalt: - case INS_sve_bfmlslb: - case INS_sve_bfmlslt: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_HB_3A; + case INS_sve_frecpe: + case INS_sve_frsqrte: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_HF_2A; break; - case INS_sve_bfmmla: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_HD_3A; + case INS_sve_sunpkhi: + case INS_sve_sunpklo: + case INS_sve_uunpkhi: + case INS_sve_uunpklo: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_CH_2A; break; - case INS_sve_fmmla: - unreached(); // TODO-SVE: Not yet supported. - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_HD_3A_A; + case INS_sve_fexpa: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_BJ_2A; break; - case INS_sve_fmlallbb: - case INS_sve_fmlallbt: - case INS_sve_fmlalltb: - case INS_sve_fmlalltt: - unreached(); // TODO-SVE: Not yet supported. - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_GO_3A; - break; - - case INS_sve_bfclamp: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_GW_3B; - break; - - case INS_sve_bfdot: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_HA_3A; - break; - - case INS_sve_fdot: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_H) - { - fmt = IF_SVE_HA_3A; - } - else if (opt == INS_OPTS_SCALABLE_B) + case INS_sve_dup: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrSP(reg2)); +#ifdef DEBUG + if (opt == INS_OPTS_SCALABLE_D) { - unreached(); // TODO-SVE: Not yet supported. - fmt = IF_SVE_HA_3A_E; + assert(size == EA_8BYTE); } else { - unreached(); // TODO-SVE: Not yet supported. - assert(insOptsNone(opt)); - fmt = IF_SVE_HA_3A_F; + assert(size == EA_4BYTE); } - break; - - case INS_sve_eorbt: - case INS_sve_eortb: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FP_3A; - break; +#endif // DEBUG + reg2 = encodingSPtoZR(reg2); + fmt = IF_SVE_CB_2A; - case INS_sve_bext: - case INS_sve_bdep: - case INS_sve_bgrp: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FQ_3A; + // DUP is an alias for MOV; + // MOV is the preferred disassembly + ins = INS_sve_mov; break; - case INS_sve_saddlbt: - case INS_sve_ssublbt: - case INS_sve_ssubltb: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FS_3A; + case INS_sve_bf1cvt: + case INS_sve_bf1cvtlt: + case INS_sve_bf2cvt: + case INS_sve_bf2cvtlt: + case INS_sve_f1cvt: + case INS_sve_f1cvtlt: + case INS_sve_f2cvt: + case INS_sve_f2cvtlt: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_HH_2A; + unreached(); // not supported yet break; - case INS_sve_saba: - case INS_sve_uaba: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FW_3A; + case INS_sve_movprfx: + assert(insScalableOptsNone(sopt)); + assert(insOptsNone(opt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_BI_2A; break; - case INS_sve_sabalb: - case INS_sve_sabalt: - case INS_sve_uabalb: - case INS_sve_uabalt: + case INS_sve_fmov: assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + assert(isPredicateRegister(reg2)); // gggg assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FX_3A; - break; + fmt = IF_SVE_BV_2B; - case INS_sve_addhnb: - case INS_sve_addhnt: - case INS_sve_raddhnb: - case INS_sve_raddhnt: - case INS_sve_subhnb: - case INS_sve_subhnt: - case INS_sve_rsubhnb: - case INS_sve_rsubhnt: - unreached(); // TODO-SVE: Not yet supported. - assert(insOptsScalableWide(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_GC_3A; + // CPY is an alias for FMOV, and MOV is an alias for CPY. + // Thus, MOV is the preferred disassembly. + ins = INS_sve_mov; break; - case INS_sve_histseg: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_GF_3A; + default: + unreached(); break; + } - case INS_sve_fclamp: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_GW_3A; - break; + assert(fmt != IF_NONE); - case INS_sve_clz: - case INS_sve_cls: - case INS_sve_cnt: - case INS_sve_cnot: - case INS_sve_not: - case INS_sve_nots: - if (isPredicateRegister(reg1) && sopt != INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // NNNN - fmt = IF_SVE_CZ_4A; - } - else - { - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AP_3A; - } - break; + instrDesc* id; - case INS_sve_fabs: - case INS_sve_fneg: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AP_3A; - break; + if (insScalableOptsWithVectorLength(sopt)) + { + id = emitNewInstr(attr); + id->idVectorLength4x(sopt == INS_SCALABLE_OPTS_VL_4X); + } + else + { + id = emitNewInstrSmall(attr); + } - case INS_sve_abs: - case INS_sve_neg: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AQ_3A; - break; + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); - case INS_sve_sxtb: - case INS_sve_uxtb: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AQ_3A; - break; + id->idReg1(reg1); + id->idReg2(reg2); - case INS_sve_sxth: - case INS_sve_uxth: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableWords(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AQ_3A; - break; + dispIns(id); + appendToCurIG(id); +} - case INS_sve_sxtw: - case INS_sve_uxtw: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AQ_3A; - break; +/***************************************************************************** + * + * Add a SVE instruction referencing a register and two constants. + */ - case INS_sve_index: - assert(isValidScalarDatasize(size)); - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert(isGeneralRegisterOrZR(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BA_3A; - break; +void emitter::emitInsSve_R_I_I( + instruction ins, emitAttr attr, regNumber reg, ssize_t imm1, ssize_t imm2, insOpts opt /* = INS_OPTS_NONE */) +{ + insFormat fmt; + ssize_t immOut; - case INS_sve_sqdmulh: - case INS_sve_sqrdmulh: - assert(isScalableVectorSize(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BE_3A; - break; + if (ins == INS_sve_index) + { + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg)); // ddddd + assert(isValidSimm<5>(imm1)); // iiiii + assert(isValidSimm<5>(imm2)); // iiiii + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + immOut = insSveEncodeTwoSimm5(imm1, imm2); + fmt = IF_SVE_AX_1A; + } + else + { + unreached(); + } - case INS_sve_ftssel: + instrDesc* id = emitNewInstrSC(attr, immOut); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing two registers and a constant. + */ + +void emitter::emitInsSve_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +{ + emitAttr size = EA_SIZE(attr); + bool hasShift = false; + insFormat fmt; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + bool isRightShift; + + case INS_sve_asr: + case INS_sve_lsl: + case INS_sve_lsr: + case INS_sve_srshr: + case INS_sve_sqshl: + case INS_sve_urshr: + case INS_sve_sqshlu: + case INS_sve_uqshl: + case INS_sve_asrd: + isRightShift = emitInsIsVectorRightShift(ins); + assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); + assert(insOptsScalableStandard(opt)); assert(isScalableVectorSize(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BK_3A; - break; - case INS_sve_compact: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableWords(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_CL_3A; + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert((ins == INS_sve_asr) || (ins == INS_sve_lsl) || (ins == INS_sve_lsr)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_SVE_BF_2A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_AM_2A; + } break; - case INS_sve_clasta: - case INS_sve_clastb: + case INS_sve_xar: assert(insOptsScalableStandard(opt)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - if (isGeneralRegister(reg1)) + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx xx + + switch (opt) { - assert(insScalableOptsNone(sopt)); - assert(isValidScalarDatasize(size)); - fmt = IF_SVE_CO_3A; + case INS_OPTS_SCALABLE_B: + assert(isValidUimmFrom1<3>(imm)); // iii + break; + + case INS_OPTS_SCALABLE_H: + assert(isValidUimmFrom1<4>(imm)); // xiii + break; + + case INS_OPTS_SCALABLE_S: + assert(isValidUimmFrom1<5>(imm)); // xxiii + break; + + case INS_OPTS_SCALABLE_D: + assert(isValidUimmFrom1<6>(imm)); // x xxiii + break; + + default: + unreached(); + break; } - else if (sopt == INS_SCALABLE_OPTS_WITH_SIMD_SCALAR) + + fmt = IF_SVE_AW_2A; + break; + + case INS_sve_index: + assert(insOptsScalable(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isValidSimm<5>(imm)); // iiiii + assert(isIntegerRegister(reg2)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + if (sopt == INS_SCALABLE_OPTS_IMM_FIRST) { - assert(isFloatReg(reg1)); - assert(isValidVectorElemsize(size)); - fmt = IF_SVE_CN_3A; + fmt = IF_SVE_AY_2A; } else { assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - fmt = IF_SVE_CM_3A; + fmt = IF_SVE_AZ_2A; } break; - case INS_sve_cpy: + case INS_sve_addvl: + case INS_sve_addpl: + assert(insOptsNone(opt)); + assert(size == EA_8BYTE); + assert(isGeneralRegisterOrSP(reg1)); // ddddd + assert(isGeneralRegisterOrSP(reg2)); // nnnnn + assert(isValidSimm<6>(imm)); // iiiiii + reg1 = encodingSPtoZR(reg1); + reg2 = encodingSPtoZR(reg2); + fmt = IF_SVE_BB_2A; + break; + case INS_sve_mov: + if (sopt == INS_SCALABLE_OPTS_BROADCAST) + { + return emitInsSve_R_R_I(INS_sve_dup, attr, reg1, reg2, imm, opt, sopt); + } + FALLTHROUGH; + case INS_sve_cpy: assert(insOptsScalableStandard(opt)); - // TODO-SVE: Following checks can be simplified to check reg1 as predicate register only after adding - // definitions for predicate registers. Currently, predicate registers P0 to P15 are aliased to simd - // registers V0 to V15. - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + assert(isVectorRegister(reg1)); // DDDDD + assert(isPredicateRegister(reg2)); // GGGG + + if (!isValidSimm<8>(imm)) { - assert(ins == INS_sve_mov); - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_AU_3A; - // ORR is an alias for MOV, and is always the preferred disassembly. - ins = INS_sve_orr; + // Size specifier must be able to fit a left-shifted immediate + assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii + assert(insOptsScalableAtLeastHalf(opt)); + hasShift = true; + imm >>= 8; } - else if (isPredicateRegister(reg3) && - (sopt == INS_SCALABLE_OPTS_NONE || sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE)) + + if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // NNNN - fmt = sopt == INS_SCALABLE_OPTS_NONE ? IF_SVE_CZ_4A : IF_SVE_CZ_4A_K; - // MOV is an alias for CPY, and is always the preferred disassembly. - ins = INS_sve_mov; + fmt = IF_SVE_BV_2A_J; } - else if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) + else { - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_CW_4A; + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_BV_2A; + } + + // MOV is an alias for CPY, and is always the preferred disassembly. + ins = INS_sve_mov; + break; + + case INS_sve_dup: + assert(insOptsScalable(opt)); + assert(isVectorRegister(reg1)); // DDDDD + assert(isVectorRegister(reg2)); // GGGG + assert(isValidBroadcastImm(imm, optGetSveElemsize(opt))); + fmt = IF_SVE_BW_2A; + ins = INS_sve_mov; // Set preferred alias for disassembly + break; + + case INS_sve_pmov: + if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) + { + assert(isPredicateRegister(reg1)); + assert(isVectorRegister(reg2)); + switch (opt) + { + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<3>(imm)); + fmt = IF_SVE_CE_2B; + break; + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + fmt = IF_SVE_CE_2D; + break; + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<1>(imm)); + fmt = IF_SVE_CE_2C; + break; + default: + unreached(); + } + } + else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) + { + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + switch (opt) + { + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<3>(imm)); + fmt = IF_SVE_CF_2B; + break; + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + fmt = IF_SVE_CF_2D; + break; + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<1>(imm)); + fmt = IF_SVE_CF_2C; + break; + default: + unreached(); + } + } + else + { + unreached(); + } + break; + + case INS_sve_sqrshrn: + case INS_sve_sqrshrun: + case INS_sve_uqrshrn: + isRightShift = emitInsIsVectorRightShift(ins); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(opt == INS_OPTS_SCALABLE_H); + assert(isRightShift); // These are always right-shift. + assert(isValidVectorShiftAmount(imm, EA_4BYTE, isRightShift)); + fmt = IF_SVE_GA_2A; + break; + + case INS_sve_pext: + assert(insOptsScalableStandard(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isHighPredicateRegister(reg2)); // NNN + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + if (sopt == INS_SCALABLE_OPTS_WITH_PREDICATE_PAIR) + { + assert(isValidUimm<1>(imm)); // i + fmt = IF_SVE_DW_2B; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_DW_2A; + } + break; + + case INS_sve_sshllb: + case INS_sve_sshllt: + case INS_sve_ushllb: + case INS_sve_ushllt: + assert(insOptsScalableWide(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // x xx + + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimm<3>(imm)); // iii + break; + + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<4>(imm)); // x iii + break; + + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<5>(imm)); // xx iii + break; + + default: + unreached(); + break; + } + + fmt = IF_SVE_FR_2A; + break; + + case INS_sve_sqshrunb: + case INS_sve_sqshrunt: + case INS_sve_sqrshrunb: + case INS_sve_sqrshrunt: + case INS_sve_shrnb: + case INS_sve_shrnt: + case INS_sve_rshrnb: + case INS_sve_rshrnt: + case INS_sve_sqshrnb: + case INS_sve_sqshrnt: + case INS_sve_sqrshrnb: + case INS_sve_sqrshrnt: + case INS_sve_uqshrnb: + case INS_sve_uqshrnt: + case INS_sve_uqrshrnb: + case INS_sve_uqrshrnt: + assert(insOptsScalableWide(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // x xx + + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimmFrom1<3>(imm)); // iii + break; + + case INS_OPTS_SCALABLE_H: + assert(isValidUimmFrom1<4>(imm)); // x iii + break; + + case INS_OPTS_SCALABLE_S: + assert(isValidUimmFrom1<5>(imm)); // xx iii + break; + + default: + unreached(); + break; + } + + fmt = IF_SVE_GB_2A; + break; + + case INS_sve_cadd: + case INS_sve_sqcadd: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + // Convert rot to bitwise representation: 0 if 90, 1 if 270 + imm = emitEncodeRotationImm90_or_270(imm); // r + fmt = IF_SVE_FV_2A; + break; + + case INS_sve_ftmad: + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isValidUimm<3>(imm)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_HN_2A; + break; + + case INS_sve_ldr: + assert(insOptsNone(opt)); + assert(isScalableVectorSize(size)); + assert(isGeneralRegister(reg2)); // nnnnn + assert(isValidSimm<9>(imm)); // iii + // iiiiii + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg1)); + fmt = IF_SVE_IE_2A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isPredicateRegister(reg1)); + fmt = IF_SVE_ID_2A; + } + break; + + case INS_sve_str: + assert(insOptsNone(opt)); + assert(isScalableVectorSize(size)); + assert(isGeneralRegister(reg2)); // nnnnn + assert(isValidSimm<9>(imm)); // iii + // iiiiii + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg1)); + fmt = IF_SVE_JH_2A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isPredicateRegister(reg1)); + fmt = IF_SVE_JG_2A; + } + break; + + case INS_sve_sli: + case INS_sve_sri: + isRightShift = emitInsIsVectorRightShift(ins); + assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_FT_2A; + break; + + case INS_sve_srsra: + case INS_sve_ssra: + case INS_sve_ursra: + case INS_sve_usra: + isRightShift = emitInsIsVectorRightShift(ins); + assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_FU_2A; + break; + + case INS_sve_ext: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isValidUimm<8>(imm)); // iiiii iii + + if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) + { + fmt = IF_SVE_BQ_2A; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_BQ_2B; + } + break; + + case INS_sve_dupq: + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); +#ifdef DEBUG + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimm<4>(imm)); + break; + + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<3>(imm)); + break; + + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + break; + + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<1>(imm)); + break; + + default: + break; + } +#endif // DEBUG + fmt = IF_SVE_BX_2A; + break; + + case INS_sve_extq: + assert(opt == INS_OPTS_SCALABLE_B); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + assert(isValidUimm<4>(imm)); + fmt = IF_SVE_BY_2A; + break; + + default: + unreached(); + break; + } + + // For encodings with shifted immediates, we need a way to determine if the immediate has been shifted or not. + // We could just leave the immediate in its unshifted form, and call emitNewInstrSC, + // but that would allocate unnecessarily large descriptors. Therefore: + // - For encodings without any shifting, just call emitNewInstrSC. + // - For unshifted immediates, call emitNewInstrSC. + // If it allocates a small descriptor, idHasShift() will always return false. + // Else, idHasShift still returns false, as we set the dedicated bit in large descriptors to false. + // - For immediates that need a shift, call emitNewInstrCns so a normal or large descriptor is used. + // idHasShift will always check the dedicated bit, as it is always available. We set this bit to true below. + instrDesc* id = !hasShift ? emitNewInstrSC(attr, imm) : emitNewInstrCns(attr, imm); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg1); + id->idReg2(reg2); + + id->idHasShift(hasShift); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing two registers and a floating point constant. + */ + +void emitter::emitInsSve_R_R_F( + instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, double immDbl, insOpts opt /* = INS_OPTS_NONE */) +{ + ssize_t imm = 0; + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_fmul: + case INS_sve_fmaxnm: + case INS_sve_fadd: + case INS_sve_fmax: + case INS_sve_fminnm: + case INS_sve_fsub: + case INS_sve_fmin: + case INS_sve_fsubr: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isScalableVectorSize(size)); + imm = emitEncodeSmallFloatImm(immDbl, ins); + fmt = IF_SVE_HM_2A; + break; + + case INS_sve_fmov: + case INS_sve_fcpy: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isPredicateRegister(reg2)); // gggg + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + floatImm8 fpi; + fpi.immFPIVal = 0; + canEncodeFloatImm8(immDbl, &fpi); + imm = fpi.immFPIVal; + fmt = IF_SVE_BU_2A; + + // FMOV is an alias for FCPY, and is always the preferred disassembly. + ins = INS_sve_fmov; + break; + + default: + unreached(); + break; + + } // end switch (ins) + + assert(fmt != IF_NONE); + + instrDesc* id = emitNewInstrSC(attr, imm); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg1); + id->idReg2(reg2); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing three registers. + */ + +void emitter::emitInsSve_R_R_R(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +{ + emitAttr size = EA_SIZE(attr); + bool pmerge = false; + bool vectorLength4x = false; + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_and: + case INS_sve_bic: + case INS_sve_eor: + case INS_sve_orr: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // mmmmm + assert(isVectorRegister(reg3)); // ddddd + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg2)); // nnnnn + fmt = IF_SVE_AU_3A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_AA_3A; + } + break; + + case INS_sve_add: + case INS_sve_sub: + case INS_sve_subr: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); + assert(ins != INS_sve_subr); + fmt = IF_SVE_AT_3A; + } + else + { + assert(isLowPredicateRegister(reg2)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AB_3A; + } + break; + + case INS_sve_addpt: + case INS_sve_subpt: + unreached(); // TODO-SVE: Not yet supported. + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg3)); // mmmmm + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); // nnnnn + fmt = IF_SVE_AT_3B; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_AB_3B; + } + break; + + case INS_sve_sdiv: + case INS_sve_sdivr: + case INS_sve_udiv: + case INS_sve_udivr: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableWords(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AC_3A; + break; + + case INS_sve_sabd: + case INS_sve_smax: + case INS_sve_smin: + case INS_sve_uabd: + case INS_sve_umax: + case INS_sve_umin: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AD_3A; + break; + + case INS_sve_mul: + case INS_sve_smulh: + case INS_sve_umulh: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); + fmt = IF_SVE_BD_3A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); + fmt = IF_SVE_AE_3A; + } + break; + + case INS_sve_pmul: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_BD_3B; + break; + + case INS_sve_andv: + case INS_sve_eorv: + case INS_sve_orv: + assert(isFloatReg(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AF_3A; + break; + + case INS_sve_andqv: + case INS_sve_eorqv: + case INS_sve_orqv: + unreached(); // TODO-SVE: Not yet supported. + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AG_3A; + break; + + case INS_sve_movprfx: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) + { + pmerge = true; + } + fmt = IF_SVE_AH_3A; + break; + + case INS_sve_saddv: + case INS_sve_uaddv: + assert(isFloatReg(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableWide(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AI_3A; + break; + + case INS_sve_addqv: + unreached(); // TODO-SVE: Not yet supported. + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AJ_3A; + break; + + case INS_sve_smaxv: + case INS_sve_sminv: + case INS_sve_umaxv: + case INS_sve_uminv: + assert(isFloatReg(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AK_3A; + break; + + case INS_sve_smaxqv: + case INS_sve_sminqv: + case INS_sve_umaxqv: + case INS_sve_uminqv: + unreached(); // TODO-SVE: Not yet supported. + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AL_3A; + break; + + case INS_sve_asrr: + case INS_sve_lslr: + case INS_sve_lsrr: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AN_3A; + break; + + case INS_sve_asr: + case INS_sve_lsl: + case INS_sve_lsr: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg3)); + if (sopt == INS_SCALABLE_OPTS_WIDE) + { + assert(isLowPredicateRegister(reg2)); + assert(insOptsScalableWide(opt)); + fmt = IF_SVE_AO_3A; + } + else if (sopt == INS_SCALABLE_OPTS_UNPREDICATED_WIDE) + { + assert(isVectorRegister(reg2)); + assert(insOptsScalableWide(opt)); + fmt = IF_SVE_BG_3A; + } + else + { + assert(isLowPredicateRegister(reg2)); + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_AN_3A; + } + break; + + case INS_sve_uzp1: + case INS_sve_trn1: + case INS_sve_zip1: + case INS_sve_uzp2: + case INS_sve_trn2: + case INS_sve_zip2: + assert(insOptsScalable(opt)); + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_Q) + { + fmt = IF_SVE_BR_3B; + } + else + { + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_BR_3A; + } + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // NNNN + assert(isPredicateRegister(reg3)); // MMMM + fmt = IF_SVE_CI_3A; + } + break; + + case INS_sve_tbl: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) + { + fmt = IF_SVE_BZ_3A_A; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_BZ_3A; + } + break; + + case INS_sve_tbx: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_BZ_3A; + break; + + case INS_sve_tbxq: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_CA_3A; + break; + + case INS_sve_sdot: + case INS_sve_udot: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + fmt = IF_SVE_EF_3A; + } + else + { + fmt = IF_SVE_EH_3A; + assert(insOptsScalableWords(opt)); + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + } + break; + + case INS_sve_usdot: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_EI_3A; + break; + + case INS_sve_smlalb: + case INS_sve_smlalt: + case INS_sve_umlalb: + case INS_sve_umlalt: + case INS_sve_smlslb: + case INS_sve_smlslt: + case INS_sve_umlslb: + case INS_sve_umlslt: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_EL_3A; + break; + + case INS_sve_sqrdmlah: + case INS_sve_sqrdmlsh: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_EM_3A; + break; + + case INS_sve_sqdmlalbt: + case INS_sve_sqdmlslbt: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_EN_3A; + break; + + case INS_sve_sqdmlalb: + case INS_sve_sqdmlalt: + case INS_sve_sqdmlslb: + case INS_sve_sqdmlslt: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_EO_3A; + break; + + case INS_sve_sclamp: + case INS_sve_uclamp: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_EV_3A; + break; + + case INS_sve_zipq1: + case INS_sve_zipq2: + case INS_sve_uzpq1: + case INS_sve_uzpq2: + case INS_sve_tblq: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_EX_3A; + break; + + case INS_sve_saddlb: + case INS_sve_saddlt: + case INS_sve_uaddlb: + case INS_sve_uaddlt: + case INS_sve_ssublb: + case INS_sve_ssublt: + case INS_sve_usublb: + case INS_sve_usublt: + case INS_sve_sabdlb: + case INS_sve_sabdlt: + case INS_sve_uabdlb: + case INS_sve_uabdlt: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FL_3A; + break; + + case INS_sve_saddwb: + case INS_sve_saddwt: + case INS_sve_uaddwb: + case INS_sve_uaddwt: + case INS_sve_ssubwb: + case INS_sve_ssubwt: + case INS_sve_usubwb: + case INS_sve_usubwt: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FM_3A; + break; + + case INS_sve_smullb: + case INS_sve_smullt: + case INS_sve_umullb: + case INS_sve_umullt: + case INS_sve_sqdmullb: + case INS_sve_sqdmullt: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FN_3A; + break; + + case INS_sve_pmullb: + case INS_sve_pmullt: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_Q) + { + fmt = IF_SVE_FN_3B; + } + else + { + assert((opt == INS_OPTS_SCALABLE_H) || (opt == INS_OPTS_SCALABLE_D)); + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FN_3A; + } + break; + + case INS_sve_smmla: + case INS_sve_usmmla: + case INS_sve_ummla: + assert(opt == INS_OPTS_SCALABLE_S); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_FO_3A; + break; + + case INS_sve_rax1: + case INS_sve_sm4ekey: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (ins == INS_sve_rax1) + { + assert(opt == INS_OPTS_SCALABLE_D); + } + else + { + assert(opt == INS_OPTS_SCALABLE_S); + } + + fmt = IF_SVE_GJ_3A; + break; + + case INS_sve_fmlalb: + case INS_sve_fmlalt: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_B) + { + unreached(); // TODO-SVE: Not yet supported. + fmt = IF_SVE_GN_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_H); + fmt = IF_SVE_HB_3A; + } + break; + + case INS_sve_fmlslb: + case INS_sve_fmlslt: + case INS_sve_bfmlalb: + case INS_sve_bfmlalt: + case INS_sve_bfmlslb: + case INS_sve_bfmlslt: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_HB_3A; + break; + + case INS_sve_bfmmla: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_HD_3A; + break; + + case INS_sve_fmmla: + unreached(); // TODO-SVE: Not yet supported. + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_HD_3A_A; + break; + + case INS_sve_fmlallbb: + case INS_sve_fmlallbt: + case INS_sve_fmlalltb: + case INS_sve_fmlalltt: + unreached(); // TODO-SVE: Not yet supported. + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_GO_3A; + break; + + case INS_sve_bfclamp: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_GW_3B; + break; + + case INS_sve_bfdot: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_HA_3A; + break; + + case INS_sve_fdot: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + fmt = IF_SVE_HA_3A; + } + else if (opt == INS_OPTS_SCALABLE_B) + { + unreached(); // TODO-SVE: Not yet supported. + fmt = IF_SVE_HA_3A_E; + } + else + { + unreached(); // TODO-SVE: Not yet supported. + assert(insOptsNone(opt)); + fmt = IF_SVE_HA_3A_F; + } + break; + + case INS_sve_eorbt: + case INS_sve_eortb: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FP_3A; + break; + + case INS_sve_bext: + case INS_sve_bdep: + case INS_sve_bgrp: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FQ_3A; + break; + + case INS_sve_saddlbt: + case INS_sve_ssublbt: + case INS_sve_ssubltb: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FS_3A; + break; + + case INS_sve_saba: + case INS_sve_uaba: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FW_3A; + break; + + case INS_sve_sabalb: + case INS_sve_sabalt: + case INS_sve_uabalb: + case INS_sve_uabalt: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_FX_3A; + break; + + case INS_sve_addhnb: + case INS_sve_addhnt: + case INS_sve_raddhnb: + case INS_sve_raddhnt: + case INS_sve_subhnb: + case INS_sve_subhnt: + case INS_sve_rsubhnb: + case INS_sve_rsubhnt: + unreached(); // TODO-SVE: Not yet supported. + assert(insOptsScalableWide(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_GC_3A; + break; + + case INS_sve_histseg: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_GF_3A; + break; + + case INS_sve_fclamp: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_GW_3A; + break; + + case INS_sve_clz: + case INS_sve_cls: + case INS_sve_cnt: + case INS_sve_cnot: + case INS_sve_not: + case INS_sve_nots: + if (isPredicateRegister(reg1) && sopt != INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // NNNN + fmt = IF_SVE_CZ_4A; + } + else + { + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AP_3A; + } + break; + + case INS_sve_fabs: + case INS_sve_fneg: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AP_3A; + break; + + case INS_sve_abs: + case INS_sve_neg: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AQ_3A; + break; + + case INS_sve_sxtb: + case INS_sve_uxtb: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AQ_3A; + break; + + case INS_sve_sxth: + case INS_sve_uxth: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableWords(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AQ_3A; + break; + + case INS_sve_sxtw: + case INS_sve_uxtw: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AQ_3A; + break; + + case INS_sve_index: + assert(isValidScalarDatasize(size)); + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrZR(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_BA_3A; + break; + + case INS_sve_sqdmulh: + case INS_sve_sqrdmulh: + assert(isScalableVectorSize(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_BE_3A; + break; + + case INS_sve_ftssel: + assert(isScalableVectorSize(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_BK_3A; + break; + + case INS_sve_compact: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableWords(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_CL_3A; + break; + + case INS_sve_clasta: + case INS_sve_clastb: + assert(insOptsScalableStandard(opt)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + if (isGeneralRegister(reg1)) + { + assert(insScalableOptsNone(sopt)); + assert(isValidScalarDatasize(size)); + fmt = IF_SVE_CO_3A; + } + else if (sopt == INS_SCALABLE_OPTS_WITH_SIMD_SCALAR) + { + assert(isFloatReg(reg1)); + assert(isValidVectorElemsize(size)); + fmt = IF_SVE_CN_3A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + fmt = IF_SVE_CM_3A; + } + break; + + case INS_sve_cpy: + case INS_sve_mov: + assert(insOptsScalableStandard(opt)); + // TODO-SVE: Following checks can be simplified to check reg1 as predicate register only after adding + // definitions for predicate registers. Currently, predicate registers P0 to P15 are aliased to simd + // registers V0 to V15. + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(ins == INS_sve_mov); + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_AU_3A; + // ORR is an alias for MOV, and is always the preferred disassembly. + ins = INS_sve_orr; + } + else if (isPredicateRegister(reg3) && + (sopt == INS_SCALABLE_OPTS_NONE || sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE)) + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // NNNN + fmt = sopt == INS_SCALABLE_OPTS_NONE ? IF_SVE_CZ_4A : IF_SVE_CZ_4A_K; + // MOV is an alias for CPY, and is always the preferred disassembly. + ins = INS_sve_mov; + } + else if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) + { + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_CW_4A; + } + else + { + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + if (isGeneralRegisterOrSP(reg3)) + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_CQ_3A; + reg3 = encodingSPtoZR(reg3); + } + else + { + assert(sopt == INS_SCALABLE_OPTS_WITH_SIMD_SCALAR); + assert(isVectorRegister(reg3)); + fmt = IF_SVE_CP_3A; + } + + // MOV is an alias for CPY, and is always the preferred disassembly. + ins = INS_sve_mov; + } + break; + + case INS_sve_lasta: + case INS_sve_lastb: + assert(insOptsScalableStandard(opt)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + if (isGeneralRegister(reg1)) + { + assert(insScalableOptsNone(sopt)); + assert(isGeneralRegister(reg1)); + fmt = IF_SVE_CS_3A; + } + else if (sopt == INS_SCALABLE_OPTS_WITH_SIMD_SCALAR) + { + assert(isVectorRegister(reg1)); + fmt = IF_SVE_CR_3A; + } + break; + + case INS_sve_revd: + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + fmt = IF_SVE_CT_3A; + break; + + case INS_sve_rbit: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_CU_3A; + break; + + case INS_sve_revb: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_CU_3A; + break; + + case INS_sve_revh: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableWords(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_CU_3A; + break; + + case INS_sve_revw: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_CU_3A; + break; + + case INS_sve_splice: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + fmt = (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) ? IF_SVE_CV_3A : IF_SVE_CV_3B; + break; + + case INS_sve_brka: + case INS_sve_brkb: + assert(isPredicateRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isPredicateRegister(reg3)); + assert(insOptsScalableStandard(opt)); + if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) + { + pmerge = true; + } + fmt = IF_SVE_DB_3A; + break; + + case INS_sve_brkas: + case INS_sve_brkbs: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isPredicateRegister(reg3)); + fmt = IF_SVE_DB_3B; + break; + + case INS_sve_brkn: + case INS_sve_brkns: + assert(insOptsScalable(opt)); + assert(isPredicateRegister(reg1)); // MMMM + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // NNNN + fmt = IF_SVE_DC_3A; + break; + + case INS_sve_cntp: + assert(size == EA_8BYTE); + assert(isGeneralRegister(reg1)); // ddddd + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // NNNN + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_DK_3A; + break; + + case INS_sve_shadd: + case INS_sve_shsub: + case INS_sve_shsubr: + case INS_sve_srhadd: + case INS_sve_uhadd: + case INS_sve_uhsub: + case INS_sve_uhsubr: + case INS_sve_urhadd: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_EP_3A; + break; + + case INS_sve_sadalp: + case INS_sve_uadalp: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_EQ_3A; + break; + + case INS_sve_addp: + case INS_sve_smaxp: + case INS_sve_sminp: + case INS_sve_umaxp: + case INS_sve_uminp: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_ER_3A; + break; + + case INS_sve_sqabs: + case INS_sve_sqneg: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_ES_3A; + break; + + case INS_sve_urecpe: + case INS_sve_ursqrte: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(opt == INS_OPTS_SCALABLE_S); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_ES_3A; + break; + + case INS_sve_sqadd: + case INS_sve_sqsub: + case INS_sve_uqadd: + case INS_sve_uqsub: + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(isScalableVectorSize(size)); + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); + fmt = IF_SVE_AT_3A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); + fmt = IF_SVE_ET_3A; + } + break; + + case INS_sve_sqsubr: + case INS_sve_suqadd: + case INS_sve_uqsubr: + case INS_sve_usqadd: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_ET_3A; + break; + + case INS_sve_sqrshl: + case INS_sve_sqrshlr: + case INS_sve_sqshl: + case INS_sve_sqshlr: + case INS_sve_srshl: + case INS_sve_srshlr: + case INS_sve_uqrshl: + case INS_sve_uqrshlr: + case INS_sve_uqshl: + case INS_sve_uqshlr: + case INS_sve_urshl: + case INS_sve_urshlr: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_EU_3A; + break; + + case INS_sve_fcvtnt: + case INS_sve_fcvtlt: + assert(insOptsConvertFloatStepwise(opt)); + FALLTHROUGH; + case INS_sve_fcvtxnt: + case INS_sve_bfcvtnt: + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + fmt = IF_SVE_GQ_3A; + break; + + case INS_sve_faddp: + case INS_sve_fmaxnmp: + case INS_sve_fmaxp: + case INS_sve_fminnmp: + case INS_sve_fminp: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_GR_3A; + break; + + case INS_sve_faddqv: + case INS_sve_fmaxnmqv: + case INS_sve_fminnmqv: + case INS_sve_fmaxqv: + case INS_sve_fminqv: + unreached(); // TODO-SVE: Not yet supported. + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_GS_3A; + break; + + case INS_sve_fmaxnmv: + case INS_sve_fmaxv: + case INS_sve_fminnmv: + case INS_sve_fminv: + case INS_sve_faddv: + assert(isFloatReg(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(isValidVectorElemsizeSveFloat(size)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HE_3A; + break; + + case INS_sve_fadda: + assert(isFloatReg(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(isValidVectorElemsizeSveFloat(size)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HJ_3A; + break; + + case INS_sve_frecps: + case INS_sve_frsqrts: + case INS_sve_ftsmul: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_HK_3A; + break; + + case INS_sve_fadd: + case INS_sve_fsub: + case INS_sve_fmul: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); // nnnnn + fmt = IF_SVE_HK_3A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_HL_3A; + } + break; + + case INS_sve_fabd: + case INS_sve_fdiv: + case INS_sve_fdivr: + case INS_sve_fmax: + case INS_sve_fmaxnm: + case INS_sve_fmin: + case INS_sve_fminnm: + case INS_sve_fmulx: + case INS_sve_fscale: + case INS_sve_fsubr: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HL_3A; + break; + + case INS_sve_famax: + case INS_sve_famin: + unreached(); // TODO-SVE: Not yet supported. + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HL_3A; + break; + + case INS_sve_bfmul: + case INS_sve_bfadd: + case INS_sve_bfsub: + case INS_sve_bfmaxnm: + case INS_sve_bfminnm: + case INS_sve_bfmax: + case INS_sve_bfmin: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg3)); // mmmmm + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); // nnnnn + fmt = IF_SVE_HK_3B; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_HL_3B; + } + break; + + case INS_sve_bsl: + case INS_sve_eor3: + case INS_sve_bcax: + case INS_sve_bsl1n: + case INS_sve_bsl2n: + case INS_sve_nbsl: + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // mmmmm + assert(isVectorRegister(reg3)); // kkkkk + fmt = IF_SVE_AV_3A; + break; + + case INS_sve_frintn: + case INS_sve_frintm: + case INS_sve_frintp: + case INS_sve_frintz: + case INS_sve_frinta: + case INS_sve_frintx: + case INS_sve_frinti: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HQ_3A; + break; + + case INS_sve_bfcvt: + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + fmt = IF_SVE_HO_3A; + break; + + case INS_sve_fcvt: + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + fmt = IF_SVE_HO_3B; + break; + + case INS_sve_fcvtx: + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + fmt = IF_SVE_HO_3C; + break; + + case INS_sve_fcvtzs: + case INS_sve_fcvtzu: + assert(insOptsScalableFloat(opt) || opt == INS_OPTS_H_TO_S || opt == INS_OPTS_H_TO_D || + opt == INS_OPTS_S_TO_D || opt == INS_OPTS_D_TO_S); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + fmt = IF_SVE_HP_3B; + break; + + case INS_sve_scvtf: + case INS_sve_ucvtf: + assert(insOptsScalableAtLeastHalf(opt) || opt == INS_OPTS_S_TO_H || opt == INS_OPTS_S_TO_D || + opt == INS_OPTS_D_TO_H || opt == INS_OPTS_D_TO_S); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + fmt = IF_SVE_HS_3A; + break; + + case INS_sve_frecpx: + case INS_sve_fsqrt: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HR_3A; + break; + + case INS_sve_whilege: + case INS_sve_whilegt: + case INS_sve_whilelt: + case INS_sve_whilele: + case INS_sve_whilehs: + case INS_sve_whilehi: + case INS_sve_whilelo: + case INS_sve_whilels: + assert(isGeneralRegister(reg2)); // nnnnn + assert(isGeneralRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + assert(insOptsScalableStandard(opt)); + + if (insScalableOptsNone(sopt)) + { + assert(isPredicateRegister(reg1)); // DDDD + assert(isValidGeneralDatasize(size)); // X + fmt = IF_SVE_DT_3A; + } + else if (insScalableOptsWithPredicatePair(sopt)) + { + assert(isLowPredicateRegister(reg1)); // DDD + assert(size == EA_8BYTE); + fmt = IF_SVE_DX_3A; + } + else + { + assert(insScalableOptsWithVectorLength(sopt)); // l + assert(isHighPredicateRegister(reg1)); // DDD + assert(size == EA_8BYTE); + vectorLength4x = (sopt == INS_SCALABLE_OPTS_VL_4X); + fmt = IF_SVE_DY_3A; + } + break; + + case INS_sve_whilewr: + case INS_sve_whilerw: + assert(insOptsScalableStandard(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isGeneralRegister(reg2)); // nnnnn + assert(size == EA_8BYTE); + assert(isGeneralRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_DU_3A; + break; + + case INS_sve_movs: + assert(insOptsScalable(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // NNNN + fmt = IF_SVE_CZ_4A; + break; + + case INS_sve_adclb: + case INS_sve_adclt: + case INS_sve_sbclb: + case INS_sve_sbclt: + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // x + fmt = IF_SVE_FY_3A; + break; + + case INS_sve_mlapt: + unreached(); // TODO-SVE: Not yet supported. + assert(insOptsNone(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_EW_3A; + break; + + case INS_sve_madpt: + unreached(); // TODO-SVE: Not yet supported. + assert(insOptsNone(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // mmmmm + assert(isVectorRegister(reg3)); // aaaaa + fmt = IF_SVE_EW_3B; + break; + + case INS_sve_fcmeq: + case INS_sve_fcmge: + case INS_sve_fcmgt: + case INS_sve_fcmlt: + case INS_sve_fcmle: + case INS_sve_fcmne: + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + assert(isPredicateRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_HI_3A; + break; + + case INS_sve_flogb: + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_HP_3A; + break; + + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ld1w: + case INS_sve_ld1d: + return emitIns_R_R_R_I(ins, size, reg1, reg2, reg3, 0, opt); + + default: + unreached(); + break; + } + + assert(fmt != IF_NONE); + + instrDesc* id = emitNewInstr(attr); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + + if (pmerge) + { + id->idPredicateReg2Merge(pmerge); + } + else if (vectorLength4x) + { + id->idVectorLength4x(vectorLength4x); + } + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing three registers and a constant. + * Do not call this directly. Use 'emitIns_R_R_R_I' instead. + */ + +void emitter::emitInsSve_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +{ + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_adr: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidUimm<2>(imm)); + switch (opt) + { + case INS_OPTS_SCALABLE_S: + case INS_OPTS_SCALABLE_D: + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + fmt = IF_SVE_BH_3A; + break; + case INS_OPTS_SCALABLE_D_SXTW: + fmt = IF_SVE_BH_3B; + break; + case INS_OPTS_SCALABLE_D_UXTW: + fmt = IF_SVE_BH_3B_A; + break; + default: + assert(!"invalid instruction"); + break; + } + break; + + case INS_sve_cmpeq: + case INS_sve_cmpgt: + case INS_sve_cmpge: + case INS_sve_cmpne: + case INS_sve_cmple: + case INS_sve_cmplt: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isValidSimm<5>(imm)); // iiiii + fmt = IF_SVE_CY_3A; + break; + + case INS_sve_cmphi: + case INS_sve_cmphs: + case INS_sve_cmplo: + case INS_sve_cmpls: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isValidUimm<7>(imm)); // iiiii + fmt = IF_SVE_CY_3B; + break; + + case INS_sve_sdot: + case INS_sve_udot: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_B) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_EY_3A; + } + else if (opt == INS_OPTS_SCALABLE_H) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_EG_3A; + } + else + { + assert(insOptsNone(opt)); + assert(isValidUimm<1>(imm)); // i + opt = INS_OPTS_SCALABLE_H; + fmt = IF_SVE_EY_3B; + } + break; + + case INS_sve_usdot: + case INS_sve_sudot: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmm + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_EZ_3A; + break; + + case INS_sve_mul: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + switch (opt) + { + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<3>(imm)); // iii + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + fmt = IF_SVE_FD_3A; + break; + + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); // ii + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + fmt = IF_SVE_FD_3B; + break; + + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<1>(imm)); // i + fmt = IF_SVE_FD_3C; + break; + + default: + unreached(); + break; + } + break; + + case INS_sve_cdot: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidRot(imm)); // rr + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + // Convert rot to bitwise representation + imm = emitEncodeRotationImm0_to_270(imm); + fmt = IF_SVE_EJ_3A; + break; + + case INS_sve_cmla: + case INS_sve_sqrdcmlah: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidRot(imm)); // rr + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + // Convert rot to bitwise representation + imm = emitEncodeRotationImm0_to_270(imm); + fmt = IF_SVE_EK_3A; + break; + + case INS_sve_ld1d: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalable(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isValidSimm<4>(imm)); + if (opt == INS_OPTS_SCALABLE_Q) + { + fmt = IF_SVE_IH_3A_A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + fmt = IF_SVE_IH_3A; + } + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 8>(imm))); + fmt = IF_SVE_IV_3A; + } + break; + + case INS_sve_ldff1d: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 8>(imm))); + fmt = IF_SVE_IV_3A; + break; + + case INS_sve_ld1w: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWordsOrQuadwords(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IH_3A_F; + } + else + { + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 4>(imm))); + fmt = IF_SVE_HX_3A_E; + } + break; + + case INS_sve_ld1sw: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IJ_3A; + } + else + { + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 4>(imm))); + fmt = IF_SVE_IV_3A; + } + break; + + case INS_sve_ldff1sw: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 4>(imm))); + fmt = IF_SVE_IV_3A; + break; + + case INS_sve_ld1sb: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isGeneralRegister(reg3)); + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IJ_3A_D; + } + else + { + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg3)); + assert(isValidUimm<5>(imm)); + fmt = IF_SVE_HX_3A_B; + } + break; + + case INS_sve_ld1b: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IJ_3A_E; + } + else + { + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg3)); + assert(isValidUimm<5>(imm)); + fmt = IF_SVE_HX_3A_B; + } + break; + + case INS_sve_ldff1b: + case INS_sve_ldff1sb: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isValidUimm<5>(imm)); + fmt = IF_SVE_HX_3A_B; + break; + + case INS_sve_ld1sh: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IJ_3A_F; + } + else + { + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 2>(imm))); + fmt = IF_SVE_HX_3A_E; + } + break; + + case INS_sve_ld1h: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IJ_3A_G; + } + else + { + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 2>(imm))); + fmt = IF_SVE_HX_3A_E; + } + break; + + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 2>(imm))); + fmt = IF_SVE_HX_3A_E; + break; + + case INS_sve_ldff1w: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert((isValidUimm_MultipleOf<5, 4>(imm))); + fmt = IF_SVE_HX_3A_E; + break; + + case INS_sve_ldnf1sw: + case INS_sve_ldnf1d: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IL_3A; + break; + + case INS_sve_ldnf1sh: + case INS_sve_ldnf1w: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IL_3A_A; + break; + + case INS_sve_ldnf1h: + case INS_sve_ldnf1sb: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IL_3A_B; + break; + + case INS_sve_ldnf1b: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm<4>(imm)); + fmt = IF_SVE_IL_3A_C; + break; + + case INS_sve_ldnt1b: + case INS_sve_ldnt1h: + case INS_sve_ldnt1w: + case INS_sve_ldnt1d: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm<4>(imm)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_ldnt1b: + assert(opt == INS_OPTS_SCALABLE_B); + break; + + case INS_sve_ldnt1h: + assert(opt == INS_OPTS_SCALABLE_H); + break; + + case INS_sve_ldnt1w: + assert(opt == INS_OPTS_SCALABLE_S); + break; + + case INS_sve_ldnt1d: + assert(opt == INS_OPTS_SCALABLE_D); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_IM_3A; + break; + + case INS_sve_ld1rqb: + case INS_sve_ld1rob: + case INS_sve_ld1rqh: + case INS_sve_ld1roh: + case INS_sve_ld1rqw: + case INS_sve_ld1row: + case INS_sve_ld1rqd: + case INS_sve_ld1rod: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_ld1rqb: + case INS_sve_ld1rqd: + case INS_sve_ld1rqh: + case INS_sve_ld1rqw: + assert((isValidSimm_MultipleOf<4, 16>(imm))); + break; + + case INS_sve_ld1rob: + case INS_sve_ld1rod: + case INS_sve_ld1roh: + case INS_sve_ld1row: + assert((isValidSimm_MultipleOf<4, 32>(imm))); + break; + + default: + assert(!"Invalid instruction"); + break; + } + + switch (ins) + { + case INS_sve_ld1rqb: + case INS_sve_ld1rob: + assert(opt == INS_OPTS_SCALABLE_B); + break; + + case INS_sve_ld1rqh: + case INS_sve_ld1roh: + assert(opt == INS_OPTS_SCALABLE_H); + break; + + case INS_sve_ld1rqw: + case INS_sve_ld1row: + assert(opt == INS_OPTS_SCALABLE_S); + break; + + case INS_sve_ld1rqd: + case INS_sve_ld1rod: + assert(opt == INS_OPTS_SCALABLE_D); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_IO_3A; + break; + + case INS_sve_ld2q: + case INS_sve_ld3q: + case INS_sve_ld4q: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_Q); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_ld2q: + assert((isValidSimm_MultipleOf<4, 2>(imm))); + break; + + case INS_sve_ld3q: + assert((isValidSimm_MultipleOf<4, 3>(imm))); + break; + + case INS_sve_ld4q: + assert((isValidSimm_MultipleOf<4, 4>(imm))); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_IQ_3A; + break; + + case INS_sve_ld2b: + case INS_sve_ld3b: + case INS_sve_ld4b: + case INS_sve_ld2h: + case INS_sve_ld3h: + case INS_sve_ld4h: + case INS_sve_ld2w: + case INS_sve_ld3w: + case INS_sve_ld4w: + case INS_sve_ld2d: + case INS_sve_ld3d: + case INS_sve_ld4d: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_ld2b: + case INS_sve_ld2h: + case INS_sve_ld2w: + case INS_sve_ld2d: + assert((isValidSimm_MultipleOf<4, 2>(imm))); + break; + + case INS_sve_ld3b: + case INS_sve_ld3h: + case INS_sve_ld3w: + case INS_sve_ld3d: + assert((isValidSimm_MultipleOf<4, 3>(imm))); + break; + + case INS_sve_ld4b: + case INS_sve_ld4h: + case INS_sve_ld4w: + case INS_sve_ld4d: + assert((isValidSimm_MultipleOf<4, 4>(imm))); + break; + + default: + assert(!"Invalid instruction"); + break; + } + + switch (ins) + { + case INS_sve_ld2b: + case INS_sve_ld3b: + case INS_sve_ld4b: + assert(opt == INS_OPTS_SCALABLE_B); + break; + + case INS_sve_ld2h: + case INS_sve_ld3h: + case INS_sve_ld4h: + assert(opt == INS_OPTS_SCALABLE_H); + break; + + case INS_sve_ld2w: + case INS_sve_ld3w: + case INS_sve_ld4w: + assert(opt == INS_OPTS_SCALABLE_S); + break; + + case INS_sve_ld2d: + case INS_sve_ld3d: + case INS_sve_ld4d: + assert(opt == INS_OPTS_SCALABLE_D); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_IS_3A; + break; + + case INS_sve_st2q: + case INS_sve_st3q: + case INS_sve_st4q: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_Q); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_st2q: + assert((isValidSimm_MultipleOf<4, 2>(imm))); + break; + + case INS_sve_st3q: + assert((isValidSimm_MultipleOf<4, 3>(imm))); + break; + + case INS_sve_st4q: + assert((isValidSimm_MultipleOf<4, 4>(imm))); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_JE_3A; + break; + + case INS_sve_stnt1b: + case INS_sve_stnt1h: + case INS_sve_stnt1w: + case INS_sve_stnt1d: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidSimm<4>(imm)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_stnt1b: + assert(opt == INS_OPTS_SCALABLE_B); + break; + + case INS_sve_stnt1h: + assert(opt == INS_OPTS_SCALABLE_H); + break; + + case INS_sve_stnt1w: + assert(opt == INS_OPTS_SCALABLE_S); + break; + + case INS_sve_stnt1d: + assert(opt == INS_OPTS_SCALABLE_D); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_JM_3A; + break; + + case INS_sve_st1w: + case INS_sve_st1d: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isValidSimm<4>(imm)); + + if (opt == INS_OPTS_SCALABLE_Q && (ins == INS_sve_st1d)) + { + fmt = IF_SVE_JN_3C_D; + } + else + { + if ((ins == INS_sve_st1w) && insOptsScalableWords(opt)) + { + fmt = IF_SVE_JN_3B; + } + else + { +#if DEBUG + if (ins == INS_sve_st1w) + { + assert(opt == INS_OPTS_SCALABLE_Q); + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + } +#endif // DEBUG + fmt = IF_SVE_JN_3C; + } + } + } + else + { + assert(isVectorRegister(reg3)); + if ((ins == INS_sve_st1w) && insOptsScalableWords(opt)) + { + assert((isValidUimm_MultipleOf<5, 4>(imm))); + fmt = IF_SVE_JI_3A_A; + } + else + { + assert(ins == INS_sve_st1d); + assert((isValidUimm_MultipleOf<5, 8>(imm))); + fmt = IF_SVE_JL_3A; + } + } + break; + + case INS_sve_st2b: + case INS_sve_st3b: + case INS_sve_st4b: + case INS_sve_st2h: + case INS_sve_st3h: + case INS_sve_st4h: + case INS_sve_st2w: + case INS_sve_st3w: + case INS_sve_st4w: + case INS_sve_st2d: + case INS_sve_st3d: + case INS_sve_st4d: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_st2b: + case INS_sve_st2h: + case INS_sve_st2w: + case INS_sve_st2d: + assert((isValidSimm_MultipleOf<4, 2>(imm))); + break; + + case INS_sve_st3b: + case INS_sve_st3h: + case INS_sve_st3w: + case INS_sve_st3d: + assert((isValidSimm_MultipleOf<4, 3>(imm))); + break; + + case INS_sve_st4b: + case INS_sve_st4h: + case INS_sve_st4w: + case INS_sve_st4d: + assert((isValidSimm_MultipleOf<4, 4>(imm))); + break; + + default: + assert(!"Invalid instruction"); + break; + } + + switch (ins) + { + case INS_sve_st2b: + case INS_sve_st3b: + case INS_sve_st4b: + assert(opt == INS_OPTS_SCALABLE_B); + break; + + case INS_sve_st2h: + case INS_sve_st3h: + case INS_sve_st4h: + assert(opt == INS_OPTS_SCALABLE_H); + break; + + case INS_sve_st2w: + case INS_sve_st3w: + case INS_sve_st4w: + assert(opt == INS_OPTS_SCALABLE_S); + break; + + case INS_sve_st2d: + case INS_sve_st3d: + case INS_sve_st4d: + assert(opt == INS_OPTS_SCALABLE_D); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_JO_3A; + break; + + case INS_sve_st1b: + case INS_sve_st1h: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + + if (isGeneralRegister(reg3)) + { + assert(isValidSimm<4>(imm)); + // st1h is reserved for scalable B + assert((ins == INS_sve_st1h) ? insOptsScalableAtLeastHalf(opt) : insOptsScalableStandard(opt)); + fmt = IF_SVE_JN_3A; + } + else + { + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg3)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_st1b: + assert(isValidUimm<5>(imm)); + break; + + case INS_sve_st1h: + assert((isValidUimm_MultipleOf<5, 2>(imm))); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_JI_3A_A; + } + break; + + case INS_sve_fmla: + case INS_sve_fmls: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_S) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_GU_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + assert(isValidUimm<1>(imm)); // i + fmt = IF_SVE_GU_3B; + } + break; + + case INS_sve_bfmla: + case INS_sve_bfmls: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmm + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); + assert(isValidUimm<3>(imm)); // i ii + fmt = IF_SVE_GU_3C; + break; + + case INS_sve_fmul: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_S) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_GX_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + assert(isValidUimm<1>(imm)); // i + fmt = IF_SVE_GX_3B; + } + break; + + case INS_sve_bfmul: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmm + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); + assert(isValidUimm<3>(imm)); // i ii + fmt = IF_SVE_GX_3C; + break; + + case INS_sve_fdot: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmm + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); + assert(isValidUimm<2>(imm)); // ii + + if (opt == INS_OPTS_SCALABLE_B) + { + unreached(); // TODO-SVE: Not yet supported. + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_GY_3B_D; + } + else if (opt == INS_OPTS_SCALABLE_H) + { + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_GY_3B; + } + else + { + unreached(); // TODO-SVE: Not yet supported. + assert(insOptsNone(opt)); + assert(isValidUimm<3>(imm)); // i ii + + // Simplify emitDispInsHelp logic by setting insOpt + opt = INS_OPTS_SCALABLE_B; + fmt = IF_SVE_GY_3A; + } + break; + + case INS_sve_bfdot: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmm + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_GY_3B; + break; + + case INS_sve_mla: + case INS_sve_mls: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<3>(imm)); // i ii + fmt = IF_SVE_FF_3A; + } + else if (opt == INS_OPTS_SCALABLE_S) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_FF_3B; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + assert(isValidUimm<1>(imm)); // i + fmt = IF_SVE_FF_3C; + } + break; + + case INS_sve_smullb: + case INS_sve_smullt: + case INS_sve_umullb: + case INS_sve_umullt: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<3>(imm)); // ii i + fmt = IF_SVE_FE_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_S); + assert(isValidUimm<2>(imm)); // i i + fmt = IF_SVE_FE_3B; + } + break; + + case INS_sve_smlalb: + case INS_sve_smlalt: + case INS_sve_umlalb: + case INS_sve_umlalt: + case INS_sve_smlslb: + case INS_sve_smlslt: + case INS_sve_umlslb: + case INS_sve_umlslt: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<3>(imm)); // ii i + fmt = IF_SVE_FG_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_S); + assert(isValidUimm<2>(imm)); // i i + fmt = IF_SVE_FG_3B; + } + break; + + case INS_sve_sqdmullb: + case INS_sve_sqdmullt: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<3>(imm)); // ii i + fmt = IF_SVE_FH_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_S); + assert(isValidUimm<2>(imm)); // i i + fmt = IF_SVE_FH_3B; + } + break; + + case INS_sve_sqdmulh: + case INS_sve_sqrdmulh: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<3>(imm)); // ii i + fmt = IF_SVE_FI_3A; + } + else if (opt == INS_OPTS_SCALABLE_S) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_FI_3B; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + assert(isValidUimm<1>(imm)); // i + fmt = IF_SVE_FI_3C; + } + break; + + case INS_sve_sqdmlalb: + case INS_sve_sqdmlalt: + case INS_sve_sqdmlslb: + case INS_sve_sqdmlslt: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<3>(imm)); // ii i + fmt = IF_SVE_FJ_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_S); + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_FJ_3B; + } + break; + + case INS_sve_sqrdmlah: + case INS_sve_sqrdmlsh: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<3>(imm)); // i ii + fmt = IF_SVE_FK_3A; + } + else if (opt == INS_OPTS_SCALABLE_S) + { + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + assert(isValidUimm<2>(imm)); // ii + fmt = IF_SVE_FK_3B; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + assert(isValidUimm<1>(imm)); // i + fmt = IF_SVE_FK_3C; + } + break; + + case INS_sve_fcadd: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isScalableVectorSize(size)); + imm = emitEncodeRotationImm90_or_270(imm); + fmt = IF_SVE_GP_3A; + break; + + case INS_sve_ld1rd: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert((isValidUimm_MultipleOf<6, 8>(imm))); + fmt = IF_SVE_IC_3A; + break; + + case INS_sve_ld1rsw: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert((isValidUimm_MultipleOf<6, 4>(imm))); + fmt = IF_SVE_IC_3A; + break; + + case INS_sve_ld1rsh: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert((isValidUimm_MultipleOf<6, 2>(imm))); + fmt = IF_SVE_IC_3A_A; + break; + + case INS_sve_ld1rw: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert((isValidUimm_MultipleOf<6, 4>(imm))); + fmt = IF_SVE_IC_3A_A; + break; + + case INS_sve_ld1rh: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert((isValidUimm_MultipleOf<6, 2>(imm))); + fmt = IF_SVE_IC_3A_B; + break; + + case INS_sve_ld1rsb: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidUimm<6>(imm)); + fmt = IF_SVE_IC_3A_B; + break; + + case INS_sve_ld1rb: + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isValidUimm<6>(imm)); + fmt = IF_SVE_IC_3A_C; + break; + + case INS_sve_fmlalb: + case INS_sve_fmlalt: + case INS_sve_fmlslb: + case INS_sve_fmlslt: + case INS_sve_bfmlalb: + case INS_sve_bfmlalt: + case INS_sve_bfmlslb: + case INS_sve_bfmlslt: + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmm + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); + assert(isValidUimm<3>(imm)); // ii i + fmt = IF_SVE_GZ_3A; + break; + + case INS_sve_luti2: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert(isValidUimm<3>(imm)); // iii + fmt = IF_SVE_GG_3B; + } + else + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isValidUimm<2>(imm)); // ii i + fmt = IF_SVE_GG_3A; + } + unreached(); + break; + + case INS_sve_luti4: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert(isValidUimm<2>(imm)); + + if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) + { + fmt = IF_SVE_GH_3B; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_GH_3B_B; + } + } + else + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(insScalableOptsNone(sopt)); + assert(isValidUimm<1>(imm)); // i + fmt = IF_SVE_GH_3A; + } + unreached(); + break; + + default: + unreached(); + break; + + } // end switch (ins) + assert(fmt != IF_NONE); + + instrDesc* id = emitNewInstrCns(attr, imm); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing three registers and two constants. + */ + +void emitter::emitInsSve_R_R_R_I_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm1, + ssize_t imm2, + insOpts opt) +{ + insFormat fmt = IF_NONE; + ssize_t imm; + + switch (ins) + { + case INS_sve_cdot: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + assert(isValidRot(imm2)); // rr + // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) + imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); + + if (opt == INS_OPTS_SCALABLE_B) + { + assert(isValidUimm<2>(imm1)); // ii + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + fmt = IF_SVE_FA_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_H); + assert(isValidUimm<1>(imm1)); // i + fmt = IF_SVE_FA_3B; + } + break; + + case INS_sve_cmla: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + assert(isValidRot(imm2)); // rr + // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) + imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); + + if (opt == INS_OPTS_SCALABLE_H) + { + assert(isValidUimm<2>(imm1)); // ii + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + fmt = IF_SVE_FB_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_S); + assert(isValidUimm<1>(imm1)); // i + fmt = IF_SVE_FB_3B; + } + break; + + case INS_sve_sqrdcmlah: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + assert(isValidRot(imm2)); // rr + // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) + imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); + + if (opt == INS_OPTS_SCALABLE_H) + { + assert(isValidUimm<2>(imm1)); // ii + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm + fmt = IF_SVE_FC_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_S); + assert(isValidUimm<1>(imm1)); // i + fmt = IF_SVE_FC_3B; + } + break; + + case INS_sve_fcmla: + assert(opt == INS_OPTS_SCALABLE_S); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isLowVectorRegister(reg3)); // mmmm + assert(isValidUimm<1>(imm1)); // i + assert(isValidRot(imm2)); // rr + + // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) + imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); + fmt = IF_SVE_GV_3A; + break; + + default: + unreached(); + break; + } + + assert(fmt != IF_NONE); + + instrDesc* id = emitNewInstrCns(attr, imm); + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing four registers. + * Do not call this directly. Use 'emitIns_R_R_R_R' instead. + */ + +void emitter::emitInsSve_R_R_R_R(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + regNumber reg4, + insOpts opt /* = INS_OPTS_NONE*/, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +{ + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_sel: + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + if (reg1 == reg4) + { + // mov is a preferred alias for sel + return emitInsSve_R_R_R(INS_sve_mov, attr, reg1, reg2, reg3, opt, + INS_SCALABLE_OPTS_PREDICATE_MERGE); + } + + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isPredicateRegister(reg2)); // VVVV + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + fmt = IF_SVE_CW_4A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // dddd + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // nnnn + assert(isPredicateRegister(reg4)); // mmmm + fmt = IF_SVE_CZ_4A; + } + break; + + case INS_sve_cmpeq: + case INS_sve_cmpgt: + case INS_sve_cmpge: + case INS_sve_cmphi: + case INS_sve_cmphs: + case INS_sve_cmpne: + case INS_sve_cmple: + case INS_sve_cmplo: + case INS_sve_cmpls: + case INS_sve_cmplt: + assert(isPredicateRegister(reg1)); // DDDD + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + assert(isScalableVectorSize(attr)); // xx + if (sopt == INS_SCALABLE_OPTS_WIDE) + { + assert(insOptsScalableWide(opt)); + fmt = IF_SVE_CX_4A_A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_CX_4A; + } + break; + + case INS_sve_and: + case INS_sve_orr: + case INS_sve_eor: + case INS_sve_ands: + case INS_sve_bic: + case INS_sve_orn: + case INS_sve_bics: + case INS_sve_eors: + case INS_sve_nor: + case INS_sve_nand: + case INS_sve_orrs: + case INS_sve_orns: + case INS_sve_nors: + case INS_sve_nands: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // dddd + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // nnnn + assert(isPredicateRegister(reg4)); // mmmm + fmt = IF_SVE_CZ_4A; + break; + + case INS_sve_brkpa: + case INS_sve_brkpb: + case INS_sve_brkpas: + case INS_sve_brkpbs: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // dddd + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // nnnn + assert(isPredicateRegister(reg4)); // mmmm + fmt = IF_SVE_DA_4A; + break; + + case INS_sve_fcmeq: + case INS_sve_fcmge: + case INS_sve_facge: + case INS_sve_fcmgt: + case INS_sve_facgt: + case INS_sve_fcmlt: + case INS_sve_fcmle: + case INS_sve_fcmne: + case INS_sve_fcmuo: + case INS_sve_facle: + case INS_sve_faclt: + assert(insOptsScalableFloat(opt)); + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + assert(isPredicateRegister(reg1)); // DDDD + assert(isLowPredicateRegister(reg2)); // ggg + assert(isScalableVectorSize(attr)); // xx + fmt = IF_SVE_HT_4A; + break; + + case INS_sve_match: + case INS_sve_nmatch: + assert(insOptsScalableAtMaxHalf(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + assert(isScalableVectorSize(attr)); // xx + fmt = IF_SVE_GE_4A; + break; + + case INS_sve_mla: + case INS_sve_mls: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + assert(isScalableVectorSize(size)); + fmt = IF_SVE_AR_4A; + break; + + case INS_sve_histcnt: + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_GI_4A; + break; + + case INS_sve_fmla: + case INS_sve_fmls: + case INS_sve_fnmla: + case INS_sve_fnmls: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_HU_4A; + break; + + case INS_sve_mad: + case INS_sve_msb: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // mmmmm + assert(isVectorRegister(reg4)); // aaaaa + assert(isScalableVectorSize(size)); + fmt = IF_SVE_AS_4A; + break; + + case INS_sve_st1b: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isScalableVectorSize(size)); + assert(insScalableOptsNone(sopt)); + + if (insOptsScalableStandard(opt)) + { + if (isGeneralRegister(reg4)) + { + fmt = IF_SVE_JD_4A; + } + else + { + assert(isVectorRegister(reg4)); + fmt = IF_SVE_JK_4B; + } + } + else + { + assert(insOptsScalable32bitExtends(opt)); + switch (opt) + { + case INS_OPTS_SCALABLE_S_UXTW: + case INS_OPTS_SCALABLE_S_SXTW: + fmt = IF_SVE_JK_4A_B; + break; + + case INS_OPTS_SCALABLE_D_UXTW: + case INS_OPTS_SCALABLE_D_SXTW: + fmt = IF_SVE_JK_4A; + break; + + default: + assert(!"Invalid options for scalable"); + break; + } + } + break; + + case INS_sve_st1h: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isScalableVectorSize(size)); + + if (insOptsScalableStandard(opt)) + { + if (sopt == INS_SCALABLE_OPTS_LSL_N) + { + if (isGeneralRegister(reg4)) + { + // st1h is reserved for scalable B + assert((ins == INS_sve_st1h) ? insOptsScalableAtLeastHalf(opt) : true); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + fmt = IF_SVE_JD_4A; + } + else + { + assert(isVectorRegister(reg4)); + fmt = IF_SVE_JJ_4B; + } + } + else + { + assert(isVectorRegister(reg4)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_JJ_4B_E; + } + } + else + { + assert(insOptsScalable32bitExtends(opt)); + switch (opt) + { + case INS_OPTS_SCALABLE_S_UXTW: + case INS_OPTS_SCALABLE_S_SXTW: + if (insScalableOptsNone(sopt)) + { + fmt = IF_SVE_JJ_4A_D; + } + else + { + assert(sopt == INS_SCALABLE_OPTS_MOD_N); + fmt = IF_SVE_JJ_4A; + } + break; + + case INS_OPTS_SCALABLE_D_UXTW: + case INS_OPTS_SCALABLE_D_SXTW: + if (insScalableOptsNone(sopt)) + { + fmt = IF_SVE_JJ_4A_C; + } + else + { + assert(sopt == INS_SCALABLE_OPTS_MOD_N); + fmt = IF_SVE_JJ_4A_B; + } + break; + + default: + assert(!"Invalid options for scalable"); + break; + } + } + break; + + case INS_sve_st1w: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isScalableVectorSize(size)); + + if (insOptsScalableStandard(opt)) + { + if (sopt == INS_SCALABLE_OPTS_LSL_N) + { + if (isGeneralRegister(reg4)) + { + fmt = IF_SVE_JD_4B; + } + else + { + assert(isVectorRegister(reg4)); + fmt = IF_SVE_JJ_4B; + } + } + else + { + assert(isVectorRegister(reg4)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_JJ_4B_E; + } + } + else if (opt == INS_OPTS_SCALABLE_Q) + { + assert(isGeneralRegister(reg4)); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + fmt = IF_SVE_JD_4C; + } + else + { + assert(insOptsScalable32bitExtends(opt)); + assert(isVectorRegister(reg4)); + switch (opt) + { + case INS_OPTS_SCALABLE_S_UXTW: + case INS_OPTS_SCALABLE_S_SXTW: + if (insScalableOptsNone(sopt)) + { + fmt = IF_SVE_JJ_4A_D; + } + else + { + assert(sopt == INS_SCALABLE_OPTS_MOD_N); + fmt = IF_SVE_JJ_4A; + } + break; + + case INS_OPTS_SCALABLE_D_UXTW: + case INS_OPTS_SCALABLE_D_SXTW: + if (insScalableOptsNone(sopt)) + { + fmt = IF_SVE_JJ_4A_C; + } + else + { + assert(sopt == INS_SCALABLE_OPTS_MOD_N); + fmt = IF_SVE_JJ_4A_B; + } + break; + + default: + assert(!"Invalid options for scalable"); + break; + } + } + break; + + case INS_sve_st1d: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isScalableVectorSize(size)); + + if (isGeneralRegister(reg4)) + { + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + if (opt == INS_OPTS_SCALABLE_Q) + { + fmt = IF_SVE_JD_4C_A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + fmt = IF_SVE_JD_4C; + } + } + else + { + assert(isVectorRegister(reg4)); + + if (opt == INS_OPTS_SCALABLE_D) + { + if (sopt == INS_SCALABLE_OPTS_LSL_N) + { + fmt = IF_SVE_JJ_4B; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_JJ_4B_C; + } + } + else + { + assert(insOptsScalable32bitExtends(opt)); + switch (opt) + { + case INS_OPTS_SCALABLE_D_UXTW: + case INS_OPTS_SCALABLE_D_SXTW: + if (sopt == INS_SCALABLE_OPTS_MOD_N) + { + fmt = IF_SVE_JJ_4A; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_JJ_4A_B; + } + break; + + default: + assert(!"Invalid options for scalable"); + break; + } + } + } + break; + + case INS_sve_ld1b: + case INS_sve_ld1sb: + case INS_sve_ldff1b: + case INS_sve_ldff1sb: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isScalableVectorSize(size)); + assert(insScalableOptsNone(sopt)); + + if (isGeneralRegisterOrZR(reg4)) + { + switch (ins) + { + case INS_sve_ldff1b: + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_IG_4A_E; + break; + + case INS_sve_ldff1sb: + assert(insOptsScalableAtLeastHalf(opt)); + fmt = IF_SVE_IG_4A_D; + break; + + case INS_sve_ld1sb: + assert(insOptsScalableAtLeastHalf(opt)); + fmt = IF_SVE_IK_4A_F; + break; + + case INS_sve_ld1b: + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_IK_4A_H; + break; + + default: + assert(!"Invalid instruction"); + break; + } + } + else + { + assert(isVectorRegister(reg4)); + + if (insOptsScalableDoubleWord32bitExtends(opt)) + { + fmt = IF_SVE_HW_4A; + } + else if (insOptsScalableSingleWord32bitExtends(opt)) + { + fmt = IF_SVE_HW_4A_A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + fmt = IF_SVE_HW_4B; + } + } + break; + + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + case INS_sve_ld1w: + case INS_sve_ldff1w: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isScalableVectorSize(size)); + + if (isGeneralRegisterOrZR(reg4)) + { + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + + switch (ins) + { + case INS_sve_ldff1h: + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_IG_4A_G; + break; + + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + assert(insOptsScalableWords(opt)); + fmt = IF_SVE_IG_4A_F; + break; + + case INS_sve_ld1w: + assert(insOptsScalableWordsOrQuadwords(opt)); + fmt = IF_SVE_II_4A_H; + break; + + case INS_sve_ld1sh: + assert(insOptsScalableWords(opt)); + fmt = IF_SVE_IK_4A_G; + break; + + case INS_sve_ld1h: + assert(insOptsScalableAtLeastHalf(opt)); + fmt = IF_SVE_IK_4A_I; + break; + + default: + assert(!"Invalid instruction"); + break; + } + } + else + { + assert(isVectorRegister(reg4)); + + if (insOptsScalableDoubleWord32bitExtends(opt)) + { + if (sopt == INS_SCALABLE_OPTS_MOD_N) + { + fmt = IF_SVE_HW_4A_A; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HW_4A_B; + } + } + else if (insOptsScalableSingleWord32bitExtends(opt)) + { + if (sopt == INS_SCALABLE_OPTS_MOD_N) + { + fmt = IF_SVE_HW_4A; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HW_4A_C; + } + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + if (sopt == INS_SCALABLE_OPTS_LSL_N) + { + fmt = IF_SVE_HW_4B; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_HW_4B_D; + } + } + } + break; + + case INS_sve_ld1d: + case INS_sve_ld1sw: + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isScalableVectorSize(size)); + + if (isGeneralRegisterOrZR(reg4)) + { + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + + if (opt == INS_OPTS_SCALABLE_Q) + { + assert(reg4 != REG_ZR); + assert(ins == INS_sve_ld1d); + fmt = IF_SVE_II_4A_B; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + + switch (ins) + { + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + fmt = IF_SVE_IG_4A; + break; + + case INS_sve_ld1d: + assert(reg4 != REG_ZR); + fmt = IF_SVE_II_4A; + break; + + case INS_sve_ld1sw: + assert(reg4 != REG_ZR); + fmt = IF_SVE_IK_4A; + break; + + default: + assert(!"Invalid instruction"); + break; + } + } } - else + else if (insOptsScalableDoubleWord32bitExtends(opt)) { - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - if (isGeneralRegisterOrSP(reg3)) + assert(isVectorRegister(reg4)); + + if (sopt == INS_SCALABLE_OPTS_MOD_N) { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_CQ_3A; - reg3 = encodingSPtoZR(reg3); + fmt = IF_SVE_IU_4A; } else { - assert(sopt == INS_SCALABLE_OPTS_WITH_SIMD_SCALAR); - assert(isVectorRegister(reg3)); - fmt = IF_SVE_CP_3A; - } + assert(insScalableOptsNone(sopt)); - // MOV is an alias for CPY, and is always the preferred disassembly. - ins = INS_sve_mov; + if (ins == INS_sve_ld1d) + { + fmt = IF_SVE_IU_4A_C; + } + else + { + fmt = IF_SVE_IU_4A_A; + } + } } - break; - - case INS_sve_lasta: - case INS_sve_lastb: - assert(insOptsScalableStandard(opt)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - if (isGeneralRegister(reg1)) + else if (sopt == INS_SCALABLE_OPTS_LSL_N) { - assert(insScalableOptsNone(sopt)); - assert(isGeneralRegister(reg1)); - fmt = IF_SVE_CS_3A; + assert(isVectorRegister(reg4)); + assert(opt == INS_OPTS_SCALABLE_D); + fmt = IF_SVE_IU_4B; } - else if (sopt == INS_SCALABLE_OPTS_WITH_SIMD_SCALAR) + else { - assert(isVectorRegister(reg1)); - fmt = IF_SVE_CR_3A; - } - break; - - case INS_sve_revd: - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_CT_3A; - break; - - case INS_sve_rbit: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_CU_3A; - break; - - case INS_sve_revb: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_CU_3A; - break; - - case INS_sve_revh: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableWords(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_CU_3A; - break; - - case INS_sve_revw: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_CU_3A; - break; - - case INS_sve_splice: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - fmt = (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) ? IF_SVE_CV_3A : IF_SVE_CV_3B; - break; + assert(isVectorRegister(reg4)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(insScalableOptsNone(sopt)); - case INS_sve_brka: - case INS_sve_brkb: - assert(isPredicateRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isPredicateRegister(reg3)); - assert(insOptsScalableStandard(opt)); - if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) - { - pmerge = true; + if (ins == INS_sve_ld1d) + { + fmt = IF_SVE_IU_4B_D; + } + else + { + fmt = IF_SVE_IU_4B_B; + } } - fmt = IF_SVE_DB_3A; break; - case INS_sve_brkas: - case INS_sve_brkbs: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); + case INS_sve_ldnt1b: + case INS_sve_ldnt1h: + case INS_sve_ldnt1w: + case INS_sve_ldnt1d: + case INS_sve_ldnt1sb: + case INS_sve_ldnt1sh: + case INS_sve_ldnt1sw: + assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); - assert(isPredicateRegister(reg3)); - fmt = IF_SVE_DB_3B; - break; - - case INS_sve_brkn: - case INS_sve_brkns: - assert(insOptsScalable(opt)); - assert(isPredicateRegister(reg1)); // MMMM - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // NNNN - fmt = IF_SVE_DC_3A; - break; + assert(isScalableVectorSize(size)); - case INS_sve_cntp: - assert(size == EA_8BYTE); - assert(isGeneralRegister(reg1)); // ddddd - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // NNNN - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_DK_3A; - break; + if (isGeneralRegister(reg3)) + { + assert(isGeneralRegister(reg4)); - case INS_sve_shadd: - case INS_sve_shsub: - case INS_sve_shsubr: - case INS_sve_srhadd: - case INS_sve_uhadd: - case INS_sve_uhsub: - case INS_sve_uhsubr: - case INS_sve_urhadd: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_EP_3A; - break; +#ifdef DEBUG + switch (ins) + { + case INS_sve_ldnt1b: + assert(opt == INS_OPTS_SCALABLE_B); + assert(insScalableOptsNone(sopt)); + break; - case INS_sve_sadalp: - case INS_sve_uadalp: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_EQ_3A; - break; + case INS_sve_ldnt1h: + assert(opt == INS_OPTS_SCALABLE_H); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; - case INS_sve_addp: - case INS_sve_smaxp: - case INS_sve_sminp: - case INS_sve_umaxp: - case INS_sve_uminp: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_ER_3A; - break; + case INS_sve_ldnt1w: + assert(opt == INS_OPTS_SCALABLE_S); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; - case INS_sve_sqabs: - case INS_sve_sqneg: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_ES_3A; - break; + case INS_sve_ldnt1d: + assert(opt == INS_OPTS_SCALABLE_D); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; - case INS_sve_urecpe: - case INS_sve_ursqrte: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(opt == INS_OPTS_SCALABLE_S); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_ES_3A; - break; + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG - case INS_sve_sqadd: - case INS_sve_sqsub: - case INS_sve_uqadd: - case INS_sve_uqsub: - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(isScalableVectorSize(size)); - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + fmt = IF_SVE_IN_4A; + } + else if ((ins == INS_sve_ldnt1d) || (ins == INS_sve_ldnt1sw)) { - assert(isVectorRegister(reg2)); - fmt = IF_SVE_AT_3A; + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg3)); + assert(isGeneralRegisterOrZR(reg4)); + assert(insScalableOptsNone(sopt)); + assert(opt == INS_OPTS_SCALABLE_D); + fmt = IF_SVE_IX_4A; } else { + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg3)); + assert(isGeneralRegisterOrZR(reg4)); assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); - fmt = IF_SVE_ET_3A; + + if (opt == INS_OPTS_SCALABLE_S) + { + fmt = IF_SVE_IF_4A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + fmt = IF_SVE_IF_4A_A; + } } break; - case INS_sve_sqsubr: - case INS_sve_suqadd: - case INS_sve_uqsubr: - case INS_sve_usqadd: + case INS_sve_ld1rob: + case INS_sve_ld1roh: + case INS_sve_ld1row: + case INS_sve_ld1rod: + case INS_sve_ld1rqb: + case INS_sve_ld1rqh: + case INS_sve_ld1rqw: + case INS_sve_ld1rqd: assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isGeneralRegister(reg4)); assert(isScalableVectorSize(size)); - fmt = IF_SVE_ET_3A; + +#ifdef DEBUG + switch (ins) + { + case INS_sve_ld1rob: + case INS_sve_ld1rqb: + assert(opt == INS_OPTS_SCALABLE_B); + assert(insScalableOptsNone(sopt)); + break; + + case INS_sve_ld1roh: + case INS_sve_ld1rqh: + assert(opt == INS_OPTS_SCALABLE_H); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + case INS_sve_ld1row: + case INS_sve_ld1rqw: + assert(opt == INS_OPTS_SCALABLE_S); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + case INS_sve_ld1rod: + case INS_sve_ld1rqd: + assert(opt == INS_OPTS_SCALABLE_D); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_IP_4A; break; - case INS_sve_sqrshl: - case INS_sve_sqrshlr: - case INS_sve_sqshl: - case INS_sve_sqshlr: - case INS_sve_srshl: - case INS_sve_srshlr: - case INS_sve_uqrshl: - case INS_sve_uqrshlr: - case INS_sve_uqshl: - case INS_sve_uqshlr: - case INS_sve_urshl: - case INS_sve_urshlr: + case INS_sve_ld1q: assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); + assert(isPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); + assert(isGeneralRegisterOrZR(reg4)); + assert(isScalableVectorSize(size)); + assert(opt == INS_OPTS_SCALABLE_Q); assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_EU_3A; + fmt = IF_SVE_IW_4A; break; - case INS_sve_fcvtnt: - case INS_sve_fcvtlt: - assert(insOptsConvertFloatStepwise(opt)); - FALLTHROUGH; - case INS_sve_fcvtxnt: - case INS_sve_bfcvtnt: - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_GQ_3A; + case INS_sve_ld2q: + case INS_sve_ld3q: + case INS_sve_ld4q: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isGeneralRegister(reg4)); + assert(isScalableVectorSize(size)); + assert(opt == INS_OPTS_SCALABLE_Q); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + fmt = IF_SVE_IR_4A; break; - case INS_sve_faddp: - case INS_sve_fmaxnmp: - case INS_sve_fmaxp: - case INS_sve_fminnmp: - case INS_sve_fminp: + case INS_sve_ld2b: + case INS_sve_ld3b: + case INS_sve_ld4b: + case INS_sve_ld2h: + case INS_sve_ld3h: + case INS_sve_ld4h: + case INS_sve_ld2w: + case INS_sve_ld3w: + case INS_sve_ld4w: + case INS_sve_ld2d: + case INS_sve_ld3d: + case INS_sve_ld4d: assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_GR_3A; + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isGeneralRegister(reg4)); + assert(isScalableVectorSize(size)); + +#ifdef DEBUG + switch (ins) + { + case INS_sve_ld2b: + case INS_sve_ld3b: + case INS_sve_ld4b: + assert(opt == INS_OPTS_SCALABLE_B); + assert(insScalableOptsNone(sopt)); + break; + + case INS_sve_ld2h: + case INS_sve_ld3h: + case INS_sve_ld4h: + assert(opt == INS_OPTS_SCALABLE_H); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + case INS_sve_ld2w: + case INS_sve_ld3w: + case INS_sve_ld4w: + assert(opt == INS_OPTS_SCALABLE_S); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + case INS_sve_ld2d: + case INS_sve_ld3d: + case INS_sve_ld4d: + assert(opt == INS_OPTS_SCALABLE_D); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + + fmt = IF_SVE_IT_4A; break; - case INS_sve_faddqv: - case INS_sve_fmaxnmqv: - case INS_sve_fminnmqv: - case INS_sve_fmaxqv: - case INS_sve_fminqv: - unreached(); // TODO-SVE: Not yet supported. + case INS_sve_st1q: assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); + assert(isPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); + assert(isGeneralRegisterOrZR(reg4)); + assert(isScalableVectorSize(size)); + assert(opt == INS_OPTS_SCALABLE_Q); assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_GS_3A; + fmt = IF_SVE_IY_4A; break; - case INS_sve_fmaxnmv: - case INS_sve_fmaxv: - case INS_sve_fminnmv: - case INS_sve_fminv: - case INS_sve_faddv: - assert(isFloatReg(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(isValidVectorElemsizeSveFloat(size)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HE_3A; - break; + case INS_sve_stnt1b: + case INS_sve_stnt1h: + case INS_sve_stnt1w: + case INS_sve_stnt1d: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isScalableVectorSize(size)); - case INS_sve_fadda: - assert(isFloatReg(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(isValidVectorElemsizeSveFloat(size)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HJ_3A; - break; + if (isGeneralRegister(reg3)) + { + assert(isGeneralRegister(reg4)); +#ifdef DEBUG + switch (ins) + { + case INS_sve_stnt1b: + assert(opt == INS_OPTS_SCALABLE_B); + assert(insScalableOptsNone(sopt)); + break; - case INS_sve_frecps: - case INS_sve_frsqrts: - case INS_sve_ftsmul: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_HK_3A; - break; + case INS_sve_stnt1h: + assert(opt == INS_OPTS_SCALABLE_H); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; - case INS_sve_fadd: - case INS_sve_fsub: - case INS_sve_fmul: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + case INS_sve_stnt1w: + assert(opt == INS_OPTS_SCALABLE_S); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg2)); // nnnnn - fmt = IF_SVE_HK_3A; + case INS_sve_stnt1d: + assert(opt == INS_OPTS_SCALABLE_D); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + fmt = IF_SVE_JB_4A; } else { + assert(isVectorRegister(reg3)); + assert(isGeneralRegisterOrZR(reg4)); + assert(isScalableVectorSize(size)); assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_HL_3A; + + if (opt == INS_OPTS_SCALABLE_S) + { + fmt = IF_SVE_IZ_4A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_D); + if (ins == INS_sve_stnt1d) + { + fmt = IF_SVE_JA_4A; + } + else + { + fmt = IF_SVE_IZ_4A_A; + } + } } break; - case INS_sve_fabd: - case INS_sve_fdiv: - case INS_sve_fdivr: - case INS_sve_fmax: - case INS_sve_fmaxnm: - case INS_sve_fmin: - case INS_sve_fminnm: - case INS_sve_fmulx: - case INS_sve_fscale: - case INS_sve_fsubr: + case INS_sve_st2b: + case INS_sve_st3b: + case INS_sve_st4b: + case INS_sve_st2h: + case INS_sve_st3h: + case INS_sve_st4h: + case INS_sve_st2w: + case INS_sve_st3w: + case INS_sve_st4w: + case INS_sve_st2d: + case INS_sve_st3d: + case INS_sve_st4d: assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HL_3A; - break; + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isGeneralRegister(reg4)); + assert(isScalableVectorSize(size)); - case INS_sve_famax: - case INS_sve_famin: - unreached(); // TODO-SVE: Not yet supported. - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HL_3A; - break; +#ifdef DEBUG + switch (ins) + { + case INS_sve_st2b: + case INS_sve_st3b: + case INS_sve_st4b: + assert(opt == INS_OPTS_SCALABLE_B); + assert(insScalableOptsNone(sopt)); + break; - case INS_sve_bfmul: - case INS_sve_bfadd: - case INS_sve_bfsub: - case INS_sve_bfmaxnm: - case INS_sve_bfminnm: - case INS_sve_bfmax: - case INS_sve_bfmin: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg3)); // mmmmm + case INS_sve_st2h: + case INS_sve_st3h: + case INS_sve_st4h: + assert(opt == INS_OPTS_SCALABLE_H); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg2)); // nnnnn - fmt = IF_SVE_HK_3B; - } - else - { - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_HL_3B; + case INS_sve_st2w: + case INS_sve_st3w: + case INS_sve_st4w: + assert(opt == INS_OPTS_SCALABLE_S); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + case INS_sve_st2d: + case INS_sve_st3d: + case INS_sve_st4d: + assert(opt == INS_OPTS_SCALABLE_D); + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + break; + + default: + assert(!"Invalid instruction"); + break; } +#endif // DEBUG + fmt = IF_SVE_JC_4A; break; - case INS_sve_bsl: - case INS_sve_eor3: - case INS_sve_bcax: - case INS_sve_bsl1n: - case INS_sve_bsl2n: - case INS_sve_nbsl: - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // mmmmm - assert(isVectorRegister(reg3)); // kkkkk - fmt = IF_SVE_AV_3A; + case INS_sve_st2q: + case INS_sve_st3q: + case INS_sve_st4q: + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isGeneralRegister(reg3)); + assert(isGeneralRegister(reg4)); + assert(isScalableVectorSize(size)); + assert(opt == INS_OPTS_SCALABLE_Q); + fmt = IF_SVE_JF_4A; break; - case INS_sve_frintn: - case INS_sve_frintm: - case INS_sve_frintp: - case INS_sve_frintz: - case INS_sve_frinta: - case INS_sve_frintx: - case INS_sve_frinti: + case INS_sve_bfmla: + case INS_sve_bfmls: + assert(opt == INS_OPTS_SCALABLE_H); + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HQ_3A; + assert(isVectorRegister(reg4)); + fmt = IF_SVE_HU_4B; break; - case INS_sve_bfcvt: - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HO_3A; + case INS_sve_fmad: + case INS_sve_fmsb: + case INS_sve_fnmad: + case INS_sve_fnmsb: + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(isVectorRegister(reg4)); + fmt = IF_SVE_HV_4A; break; - case INS_sve_fcvt: - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HO_3B; + default: + unreached(); break; + } + assert(fmt != IF_NONE); - case INS_sve_fcvtx: - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HO_3C; + // Use aliases. + switch (ins) + { + case INS_sve_cmple: + std::swap(reg3, reg4); + ins = INS_sve_cmpge; break; - - case INS_sve_fcvtzs: - case INS_sve_fcvtzu: - assert(insOptsScalableFloat(opt) || opt == INS_OPTS_H_TO_S || opt == INS_OPTS_H_TO_D || - opt == INS_OPTS_S_TO_D || opt == INS_OPTS_D_TO_S); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HP_3B; + case INS_sve_cmplo: + std::swap(reg3, reg4); + ins = INS_sve_cmphi; + break; + case INS_sve_cmpls: + std::swap(reg3, reg4); + ins = INS_sve_cmphs; + break; + case INS_sve_cmplt: + std::swap(reg3, reg4); + ins = INS_sve_cmpgt; + break; + case INS_sve_facle: + std::swap(reg3, reg4); + ins = INS_sve_facge; + break; + case INS_sve_faclt: + std::swap(reg3, reg4); + ins = INS_sve_facgt; + break; + case INS_sve_fcmle: + std::swap(reg3, reg4); + ins = INS_sve_fcmge; + break; + case INS_sve_fcmlt: + std::swap(reg3, reg4); + ins = INS_sve_fcmgt; + break; + default: break; + } + + instrDesc* id = emitNewInstr(attr); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + id->idReg4(reg4); - case INS_sve_scvtf: - case INS_sve_ucvtf: - assert(insOptsScalableAtLeastHalf(opt) || opt == INS_OPTS_S_TO_H || opt == INS_OPTS_S_TO_D || - opt == INS_OPTS_D_TO_H || opt == INS_OPTS_D_TO_S); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HS_3A; - break; + dispIns(id); + appendToCurIG(id); +} - case INS_sve_frecpx: - case INS_sve_fsqrt: +/***************************************************************************** + * + * Add a SVE instruction referencing four registers and a constant. + */ + +void emitter::emitInsSve_R_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + regNumber reg4, + ssize_t imm, + insOpts opt /* = INS_OPT_NONE*/) +{ + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_fcmla: + assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HR_3A; + assert(isVectorRegister(reg4)); + assert(isScalableVectorSize(size)); + imm = emitEncodeRotationImm0_to_270(imm); + fmt = IF_SVE_GT_4A; break; - case INS_sve_whilege: - case INS_sve_whilegt: - case INS_sve_whilelt: - case INS_sve_whilele: - case INS_sve_whilehs: - case INS_sve_whilehi: - case INS_sve_whilelo: - case INS_sve_whilels: - assert(isGeneralRegister(reg2)); // nnnnn - assert(isGeneralRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + case INS_sve_psel: + unreached(); // TODO-SVE: Not yet supported. assert(insOptsScalableStandard(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // NNNN + assert(isPredicateRegister(reg3)); // MMMM + assert(isGeneralRegister(reg4)); // vv + assert((REG_R12 <= reg4) && (reg4 <= REG_R15)); - if (insScalableOptsNone(sopt)) - { - assert(isPredicateRegister(reg1)); // DDDD - assert(isValidGeneralDatasize(size)); // X - fmt = IF_SVE_DT_3A; - } - else if (insScalableOptsWithPredicatePair(sopt)) - { - assert(isLowPredicateRegister(reg1)); // DDD - assert(size == EA_8BYTE); - fmt = IF_SVE_DX_3A; - } - else + switch (opt) { - assert(insScalableOptsWithVectorLength(sopt)); // l - assert(isHighPredicateRegister(reg1)); // DDD - assert(size == EA_8BYTE); - vectorLength4x = (sopt == INS_SCALABLE_OPTS_VL_4X); - fmt = IF_SVE_DY_3A; + case INS_OPTS_SCALABLE_B: + assert(isValidUimm<4>(imm)); + break; + + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<3>(imm)); + break; + + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + break; + + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<1>(imm)); + break; + + default: + unreached(); + break; } + + fmt = IF_SVE_DV_4A; break; - case INS_sve_whilewr: - case INS_sve_whilerw: - assert(insOptsScalableStandard(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isGeneralRegister(reg2)); // nnnnn - assert(size == EA_8BYTE); - assert(isGeneralRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_DU_3A; + default: + unreached(); break; + } + assert(fmt != IF_NONE); - case INS_sve_movs: - assert(insOptsScalable(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // NNNN - fmt = IF_SVE_CZ_4A; + instrDesc* id = emitNewInstrCns(attr, imm); + + id->idIns(ins); + id->idInsFmt(fmt); + id->idInsOpt(opt); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + id->idReg4(reg4); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing a register, a SVE Pattern. + */ + +void emitter::emitIns_R_PATTERN( + instruction ins, emitAttr attr, regNumber reg1, insOpts opt, insSvePattern pattern /* = SVE_PATTERN_ALL*/) +{ + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_ptrue: + case INS_sve_ptrues: + assert(isPredicateRegister(reg1)); + assert(isScalableVectorSize(attr)); + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_DE_1A; break; - case INS_sve_adclb: - case INS_sve_adclt: - case INS_sve_sbclb: - case INS_sve_sbclt: - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // x - fmt = IF_SVE_FY_3A; + default: + unreached(); break; - case INS_sve_mlapt: - unreached(); // TODO-SVE: Not yet supported. + } // end switch (ins) + assert(fmt != IF_NONE); + + instrDesc* id = emitNewInstr(attr); + + id->idIns(ins); + id->idInsFmt(fmt); + + id->idReg1(reg1); + id->idInsOpt(opt); + id->idSvePattern(pattern); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing a register, a SVE Pattern and an immediate. + */ + +void emitter::emitIns_R_PATTERN_I(instruction ins, + emitAttr attr, + regNumber reg1, + insSvePattern pattern, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */) +{ + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_cntb: + case INS_sve_cntd: + case INS_sve_cnth: + case INS_sve_cntw: assert(insOptsNone(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_EW_3A; + assert(isGeneralRegister(reg1)); // ddddd + assert(isValidUimmFrom1<4>(imm)); // iiii + assert(size == EA_8BYTE); + fmt = IF_SVE_BL_1A; break; - case INS_sve_madpt: - unreached(); // TODO-SVE: Not yet supported. - assert(insOptsNone(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // mmmmm - assert(isVectorRegister(reg3)); // aaaaa - fmt = IF_SVE_EW_3B; + case INS_sve_incd: + case INS_sve_inch: + case INS_sve_incw: + case INS_sve_decd: + case INS_sve_dech: + case INS_sve_decw: + assert(isValidUimmFrom1<4>(imm)); // iiii + + if (insOptsNone(opt)) + { + assert(isGeneralRegister(reg1)); // ddddd + assert(size == EA_8BYTE); + fmt = IF_SVE_BM_1A; + } + else + { + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + fmt = IF_SVE_BN_1A; + } break; - case INS_sve_fcmeq: - case INS_sve_fcmge: - case INS_sve_fcmgt: - case INS_sve_fcmlt: - case INS_sve_fcmle: - case INS_sve_fcmne: - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - assert(isPredicateRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_HI_3A; + case INS_sve_incb: + case INS_sve_decb: + assert(isGeneralRegister(reg1)); // ddddd + assert(isValidUimmFrom1<4>(imm)); // iiii + assert(size == EA_8BYTE); + fmt = IF_SVE_BM_1A; break; - case INS_sve_flogb: - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_HP_3A; + case INS_sve_sqincb: + case INS_sve_uqincb: + case INS_sve_sqdecb: + case INS_sve_uqdecb: + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg1)); // ddddd + assert(isValidUimmFrom1<4>(imm)); // iiii + assert(isValidGeneralDatasize(size)); // X + fmt = IF_SVE_BO_1A; break; - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ld1w: - case INS_sve_ld1d: - return emitIns_R_R_R_I(ins, size, reg1, reg2, reg3, 0, opt); + case INS_sve_sqinch: + case INS_sve_uqinch: + case INS_sve_sqdech: + case INS_sve_uqdech: + case INS_sve_sqincw: + case INS_sve_uqincw: + case INS_sve_sqdecw: + case INS_sve_uqdecw: + case INS_sve_sqincd: + case INS_sve_uqincd: + case INS_sve_sqdecd: + case INS_sve_uqdecd: + assert(isValidUimmFrom1<4>(imm)); // iiii + + if (insOptsNone(opt)) + { + assert(isGeneralRegister(reg1)); // ddddd + assert(isValidGeneralDatasize(size)); // X + fmt = IF_SVE_BO_1A; + } + else + { + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isScalableVectorSize(size)); + fmt = IF_SVE_BP_1A; + } + break; default: unreached(); break; - } + } // end switch (ins) assert(fmt != IF_NONE); - instrDesc* id = emitNewInstr(attr); + instrDesc* id = emitNewInstrCns(attr, imm); id->idIns(ins); id->idInsFmt(fmt); id->idInsOpt(opt); + id->idOpSize(size); id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - - if (pmerge) - { - id->idPredicateReg2Merge(pmerge); - } - else if (vectorLength4x) - { - id->idVectorLength4x(vectorLength4x); - } + id->idSvePattern(pattern); dispIns(id); appendToCurIG(id); @@ -2907,3006 +7190,5437 @@ void emitter::emitInsSve_R_R_R(instruction ins, /***************************************************************************** * - * Add a SVE instruction referencing three registers and a constant. - * Do not call this directly. Use 'emitIns_R_R_R_I' instead. + * Add a SVE instruction referencing three registers and a SVE 'prfop'. */ -void emitter::emitInsSve_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +void emitter::emitIns_PRFOP_R_R_R(instruction ins, + emitAttr attr, + insSvePrfop prfop, + regNumber reg1, + regNumber reg2, + regNumber reg3, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) { - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_sve_adr: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidUimm<2>(imm)); - switch (opt) - { - case INS_OPTS_SCALABLE_S: - case INS_OPTS_SCALABLE_D: - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - fmt = IF_SVE_BH_3A; - break; - case INS_OPTS_SCALABLE_D_SXTW: - fmt = IF_SVE_BH_3B; - break; - case INS_OPTS_SCALABLE_D_UXTW: - fmt = IF_SVE_BH_3B_A; - break; - default: - assert(!"invalid instruction"); - break; - } - break; - - case INS_sve_cmpeq: - case INS_sve_cmpgt: - case INS_sve_cmpge: - case INS_sve_cmpne: - case INS_sve_cmple: - case INS_sve_cmplt: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isValidSimm<5>(imm)); // iiiii - fmt = IF_SVE_CY_3A; - break; - - case INS_sve_cmphi: - case INS_sve_cmphs: - case INS_sve_cmplo: - case INS_sve_cmpls: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isValidUimm<7>(imm)); // iiiii - fmt = IF_SVE_CY_3B; - break; - - case INS_sve_sdot: - case INS_sve_udot: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - - if (opt == INS_OPTS_SCALABLE_B) - { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_EY_3A; - } - else if (opt == INS_OPTS_SCALABLE_H) - { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_EG_3A; - } - else - { - assert(insOptsNone(opt)); - assert(isValidUimm<1>(imm)); // i - opt = INS_OPTS_SCALABLE_H; - fmt = IF_SVE_EY_3B; - } - break; - - case INS_sve_usdot: - case INS_sve_sudot: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmm - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_EZ_3A; - break; - - case INS_sve_mul: + case INS_sve_prfb: assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm + assert(isLowPredicateRegister(reg1)); + assert(isGeneralRegister(reg2)); + assert(isScalableVectorSize(size)); - switch (opt) + if (insOptsScalable32bitExtends(opt)) { - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); // iii - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - fmt = IF_SVE_FD_3A; - break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); // ii - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - fmt = IF_SVE_FD_3B; - break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_FD_3C; - break; - - default: - unreached(); - break; - } - break; - - case INS_sve_cdot: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidRot(imm)); // rr - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - // Convert rot to bitwise representation - imm = emitEncodeRotationImm0_to_270(imm); - fmt = IF_SVE_EJ_3A; - break; - - case INS_sve_cmla: - case INS_sve_sqrdcmlah: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidRot(imm)); // rr - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + assert(isVectorRegister(reg3)); - // Convert rot to bitwise representation - imm = emitEncodeRotationImm0_to_270(imm); - fmt = IF_SVE_EK_3A; + if (insOptsScalableSingleWord32bitExtends(opt)) + { + fmt = IF_SVE_HY_3A; + } + else + { + assert(insOptsScalableDoubleWord32bitExtends(opt)); + fmt = IF_SVE_HY_3A_A; + } + } + else if (isVectorRegister(reg3)) + { + assert(opt == INS_OPTS_SCALABLE_D); + fmt = IF_SVE_HY_3B; + } + else + { + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg3)); + fmt = IF_SVE_IB_3A; + } break; - case INS_sve_ld1d: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalable(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); + case INS_sve_prfh: + case INS_sve_prfw: + case INS_sve_prfd: + assert(isLowPredicateRegister(reg1)); + assert(isGeneralRegister(reg2)); + assert(isScalableVectorSize(size)); - if (isGeneralRegister(reg3)) + if (sopt == INS_SCALABLE_OPTS_MOD_N) { - assert(isValidSimm<4>(imm)); - if (opt == INS_OPTS_SCALABLE_Q) + if (insOptsScalableSingleWord32bitExtends(opt)) { - fmt = IF_SVE_IH_3A_A; + fmt = IF_SVE_HY_3A; } else { - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_IH_3A; + assert(insOptsScalableDoubleWord32bitExtends(opt)); + fmt = IF_SVE_HY_3A_A; } } else { - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 8>(imm))); - fmt = IF_SVE_IV_3A; + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + if (isVectorRegister(reg3)) + { + assert(opt == INS_OPTS_SCALABLE_D); + fmt = IF_SVE_HY_3B; + } + else + { + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg3)); + fmt = IF_SVE_IB_3A; + } } break; - case INS_sve_ldff1d: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 8>(imm))); - fmt = IF_SVE_IV_3A; + default: + unreached(); break; - case INS_sve_ld1w: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWordsOrQuadwords(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); + } // end switch (ins) + assert(fmt != IF_NONE); - if (isGeneralRegister(reg3)) - { - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IH_3A_F; - } - else + instrDesc* id = emitNewInstr(attr); + + id->idIns(ins); + id->idInsOpt(opt); + id->idInsFmt(fmt); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idReg3(reg3); + id->idSvePrfop(prfop); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Add a SVE instruction referencing two registers, a SVE 'prfop' and an immediate. + */ + +void emitter::emitIns_PRFOP_R_R_I(instruction ins, + emitAttr attr, + insSvePrfop prfop, + regNumber reg1, + regNumber reg2, + int imm, + insOpts opt /* = INS_OPTS_NONE */) +{ + emitAttr size = EA_SIZE(attr); + insFormat fmt = IF_NONE; + + /* Figure out the encoding format of the instruction */ + switch (ins) + { + case INS_sve_prfb: + case INS_sve_prfh: + case INS_sve_prfw: + case INS_sve_prfd: + assert(isLowPredicateRegister(reg1)); + assert(isScalableVectorSize(size)); + + if (isVectorRegister(reg2)) { assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 4>(imm))); - fmt = IF_SVE_HX_3A_E; - } - break; - case INS_sve_ld1sw: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); +#ifdef DEBUG + switch (ins) + { + case INS_sve_prfb: + assert(isValidUimm<5>(imm)); + break; - if (isGeneralRegister(reg3)) - { - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IJ_3A; + case INS_sve_prfh: + assert((isValidUimm_MultipleOf<5, 2>(imm))); + break; + + case INS_sve_prfw: + assert((isValidUimm_MultipleOf<5, 4>(imm))); + break; + + case INS_sve_prfd: + assert((isValidUimm_MultipleOf<5, 8>(imm))); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + fmt = IF_SVE_HZ_2A_B; } else { - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 4>(imm))); - fmt = IF_SVE_IV_3A; + assert(insOptsNone(opt)); + assert(isGeneralRegister(reg2)); + assert(isValidSimm<6>(imm)); + fmt = IF_SVE_IA_2A; } break; - case INS_sve_ldff1sw: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 4>(imm))); - fmt = IF_SVE_IV_3A; - break; + default: + unreached(); + break; + + } // end switch (ins) + assert(fmt != IF_NONE); + + instrDesc* id = emitNewInstrCns(attr, imm); + + id->idIns(ins); + id->idInsOpt(opt); + id->idInsFmt(fmt); + + id->idReg1(reg1); + id->idReg2(reg2); + id->idSvePrfop(prfop); + + dispIns(id); + appendToCurIG(id); +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize(emitAttr size) +{ + switch (size) + { + case EA_1BYTE: + return 0x00000000; + + case EA_2BYTE: + return 0x00400000; // set the bit at location 22 + + case EA_4BYTE: + return 0x00800000; // set the bit at location 23 + + case EA_8BYTE: + return 0x00C00000; // set the bit at location 23 and 22 + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the size at bit locations '22-21'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_22_to_21(emitAttr size) +{ + switch (size) + { + case EA_1BYTE: + return 0; + + case EA_2BYTE: + return (1 << 21); // set the bit at location 21 + + case EA_4BYTE: + return (1 << 22); // set the bit at location 22 + + case EA_8BYTE: + return (1 << 22) | (1 << 21); // set the bit at location 22 and 21 + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the size at bit locations '18-17'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_18_to_17(emitAttr size) +{ + switch (size) + { + case EA_1BYTE: + return 0; + + case EA_2BYTE: + return (1 << 17); // set the bit at location 17 + + case EA_4BYTE: + return (1 << 18); // set the bit at location 18 + + case EA_8BYTE: + return (1 << 18) | (1 << 17); // set the bit at location 18 and 17 + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the field 'sz' at bit location '20'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_20(emitAttr size) +{ + switch (size) + { + case EA_4BYTE: + return 0; + + case EA_8BYTE: + return (1 << 20); + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the field 'sz' at bit location '21'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_21(emitAttr size) +{ + switch (size) + { + case EA_4BYTE: + return 0; + + case EA_8BYTE: + return (1 << 21); + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:20-19'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_23_tszl_20_to_19(emitAttr size) +{ + switch (size) + { + case EA_1BYTE: + return 0x080000; // set the bit at location 19 + + case EA_2BYTE: + return 0x100000; // set the bit at location 20 - case INS_sve_ld1sb: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); + case EA_4BYTE: + return 0x400000; // set the bit at location 22 - if (isGeneralRegister(reg3)) - { - assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IJ_3A_D; - } - else - { - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg3)); - assert(isValidUimm<5>(imm)); - fmt = IF_SVE_HX_3A_B; - } - break; + case EA_8BYTE: + return 0x800000; // set the bit at location 23 - case INS_sve_ld1b: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); + default: + assert(!"Invalid size for vector register"); + } + return 0; +} - if (isGeneralRegister(reg3)) - { - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IJ_3A_E; - } - else +/***************************************************************************** + * + * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction at bit location '30'. + * This only works on select formats. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_30_or_21(insFormat fmt, emitAttr size) +{ + switch (fmt) + { + case IF_SVE_HX_3A_B: + case IF_SVE_HX_3A_E: + switch (size) { - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg3)); - assert(isValidUimm<5>(imm)); - fmt = IF_SVE_HX_3A_B; + case EA_4BYTE: + return 0; + + case EA_8BYTE: + return (1 << 30); + + default: + break; } - break; - case INS_sve_ldff1b: - case INS_sve_ldff1sb: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isValidUimm<5>(imm)); - fmt = IF_SVE_HX_3A_B; - break; + assert(!"Invalid size for vector register"); + return 0; - case INS_sve_ld1sh: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); + case IF_SVE_IV_3A: + assert(size == EA_8BYTE); + return 0; - if (isGeneralRegister(reg3)) - { - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IJ_3A_F; - } - else + case IF_SVE_JI_3A_A: + switch (size) { - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 2>(imm))); - fmt = IF_SVE_HX_3A_E; - } - break; + case EA_4BYTE: + return (1 << 21); - case INS_sve_ld1h: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); + case EA_8BYTE: + return 0; - if (isGeneralRegister(reg3)) - { - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IJ_3A_G; - } - else - { - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 2>(imm))); - fmt = IF_SVE_HX_3A_E; + default: + break; } - break; - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 2>(imm))); - fmt = IF_SVE_HX_3A_E; + assert(!"Invalid size for vector register"); + return 0; + + default: break; + } - case INS_sve_ldff1w: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 4>(imm))); - fmt = IF_SVE_HX_3A_E; + assert(!"Unexpected instruction format"); + return 0; +} +/***************************************************************************** + * + * Returns the encoding for the field 'i1:tszh:tszl' at bit locations '23-22:20-18'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_tszl_and_imm(const insOpts opt, const ssize_t imm) +{ + code_t encoding = 0; + + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimm<4>(imm)); + encoding = 0x040000; // set the bit at location 18 + // encode immediate at location 23-22:20-19 + encoding |= ((imm & 0b1100) << 22); + encoding |= ((imm & 0b11) << 19); break; - case INS_sve_ldnf1sw: - case INS_sve_ldnf1d: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IL_3A; + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<3>(imm)); + encoding = 0x080000; // set the bit at location 19 + // encode immediate at location 23-22:20 + encoding |= ((imm & 0b110) << 22); + encoding |= ((imm & 1) << 20); break; - case INS_sve_ldnf1sh: - case INS_sve_ldnf1w: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IL_3A_A; + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + encoding = 0x100000; // set the bit at location 20 + encoding |= (imm << 22); // encode immediate at location 23:22 break; - case INS_sve_ldnf1h: - case INS_sve_ldnf1sb: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IL_3A_B; + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<1>(imm)); + encoding = 0x400000; // set the bit at location 22 + encoding |= (imm << 23); // encode immediate at location 23 break; - case INS_sve_ldnf1b: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); - fmt = IF_SVE_IL_3A_C; + default: + assert(!"Invalid size for vector register"); break; + } - case INS_sve_ldnt1b: - case INS_sve_ldnt1h: - case INS_sve_ldnt1w: - case INS_sve_ldnt1d: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + return encoding; +} -#ifdef DEBUG - switch (ins) - { - case INS_sve_ldnt1b: - assert(opt == INS_OPTS_SCALABLE_B); - break; +/***************************************************************************** + * + * Returns the encoding for the field 'tszh:tszl:imm3' at bit locations '23-22:20-19:18-16'. + */ - case INS_sve_ldnt1h: - assert(opt == INS_OPTS_SCALABLE_H); - break; +/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithShift_tszh_tszl_imm3(const insOpts opt, + ssize_t imm, + bool isRightShift) +{ + code_t encoding = 0; - case INS_sve_ldnt1w: - assert(opt == INS_OPTS_SCALABLE_S); - break; + imm = insEncodeShiftImmediate(optGetSveElemsize(opt), isRightShift, imm); - case INS_sve_ldnt1d: - assert(opt == INS_OPTS_SCALABLE_D); - break; + switch (opt) + { + case INS_OPTS_SCALABLE_B: + imm = imm & 0b111; // bits 18-16 + encoding |= (1 << 19); // bit 19 + break; - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG + case INS_OPTS_SCALABLE_H: + imm = imm & 0b1111; // bits 19-16 + encoding |= (1 << 20); // bit 20 + break; - fmt = IF_SVE_IM_3A; + case INS_OPTS_SCALABLE_S: + imm = imm & 0b11111; // bits 20-16 + encoding |= (1 << 22); // bit 22 break; - case INS_sve_ld1rqb: - case INS_sve_ld1rob: - case INS_sve_ld1rqh: - case INS_sve_ld1roh: - case INS_sve_ld1rqw: - case INS_sve_ld1row: - case INS_sve_ld1rqd: - case INS_sve_ld1rod: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); + case INS_OPTS_SCALABLE_D: + // this gets the last bit of 'imm' and tries to set bit 22 + encoding |= ((imm >> 5) << 22); + imm = imm & 0b11111; // bits 20-16 + encoding |= (1 << 23); // bit 23 + break; -#ifdef DEBUG - switch (ins) - { - case INS_sve_ld1rqb: - case INS_sve_ld1rqd: - case INS_sve_ld1rqh: - case INS_sve_ld1rqw: - assert((isValidSimm_MultipleOf<4, 16>(imm))); - break; + default: + assert(!"Invalid size for vector register"); + break; + } - case INS_sve_ld1rob: - case INS_sve_ld1rod: - case INS_sve_ld1roh: - case INS_sve_ld1row: - assert((isValidSimm_MultipleOf<4, 32>(imm))); - break; + return (encoding | (code_t)(imm << 16)); +} - default: - assert(!"Invalid instruction"); - break; - } +/***************************************************************************** + * + * Returns the encoding for the field 'i1:tsz' at bit locations '20:19-16'. + */ - switch (ins) - { - case INS_sve_ld1rqb: - case INS_sve_ld1rob: - assert(opt == INS_OPTS_SCALABLE_B); - break; +/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithImmediate_i1_tsz(const insOpts opt, ssize_t imm) +{ + code_t encoding = 0; - case INS_sve_ld1rqh: - case INS_sve_ld1roh: - assert(opt == INS_OPTS_SCALABLE_H); - break; + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimm<4>(imm)); + encoding |= (1 << 16); // bit 16 + encoding |= (imm << 17); // bits 20-17 + break; - case INS_sve_ld1rqw: - case INS_sve_ld1row: - assert(opt == INS_OPTS_SCALABLE_S); - break; + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<3>(imm)); + encoding |= (1 << 17); // bit 17 + encoding |= (imm << 18); // bits 20-18 + break; - case INS_sve_ld1rqd: - case INS_sve_ld1rod: - assert(opt == INS_OPTS_SCALABLE_D); - break; + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + encoding |= (1 << 18); // bit 18 + encoding |= (imm << 19); // bits 20-19 + break; - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<1>(imm)); + encoding |= (1 << 19); // bit 19 + encoding |= (imm << 20); // bit 20 + break; - fmt = IF_SVE_IO_3A; + default: + assert(!"Invalid size for vector register"); break; + } - case INS_sve_ld2q: - case INS_sve_ld3q: - case INS_sve_ld4q: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_Q); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); + return encoding; +} -#ifdef DEBUG - switch (ins) - { - case INS_sve_ld2q: - assert((isValidSimm_MultipleOf<4, 2>(imm))); - break; +/***************************************************************************** + * + * Returns the encoding to select the elemsize for an Arm64 SVE vector instruction plus an immediate. + * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:9-8'. + */ - case INS_sve_ld3q: - assert((isValidSimm_MultipleOf<4, 3>(imm))); - break; +/*static*/ emitter::code_t emitter::insEncodeSveShift_23_to_22_9_to_0(emitAttr size, bool isRightShift, size_t imm) +{ + code_t encodedSize = 0; - case INS_sve_ld4q: - assert((isValidSimm_MultipleOf<4, 4>(imm))); - break; + switch (size) + { + case EA_1BYTE: + encodedSize = 0x100; // set the bit at location 8 + break; - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG + case EA_2BYTE: + encodedSize = 0x200; // set the bit at location 9 + break; - fmt = IF_SVE_IQ_3A; + case EA_4BYTE: + encodedSize = 0x400000; // set the bit at location 22 break; - case INS_sve_ld2b: - case INS_sve_ld3b: - case INS_sve_ld4b: - case INS_sve_ld2h: - case INS_sve_ld3h: - case INS_sve_ld4h: - case INS_sve_ld2w: - case INS_sve_ld3w: - case INS_sve_ld4w: - case INS_sve_ld2d: - case INS_sve_ld3d: - case INS_sve_ld4d: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); + case EA_8BYTE: + encodedSize = 0x800000; // set the bit at location 23 + break; -#ifdef DEBUG - switch (ins) - { - case INS_sve_ld2b: - case INS_sve_ld2h: - case INS_sve_ld2w: - case INS_sve_ld2d: - assert((isValidSimm_MultipleOf<4, 2>(imm))); - break; + default: + assert(!"Invalid esize for vector register"); + } - case INS_sve_ld3b: - case INS_sve_ld3h: - case INS_sve_ld3w: - case INS_sve_ld3d: - assert((isValidSimm_MultipleOf<4, 3>(imm))); - break; + code_t encodedImm = insEncodeShiftImmediate(size, isRightShift, imm); + code_t imm3High = (encodedImm & 0x60) << 17; + code_t imm3Low = (encodedImm & 0x1f) << 5; + return encodedSize | imm3High | imm3Low; +} - case INS_sve_ld4b: - case INS_sve_ld4h: - case INS_sve_ld4w: - case INS_sve_ld4d: - assert((isValidSimm_MultipleOf<4, 4>(imm))); - break; +/***************************************************************************** + * + * Returns the encoding to select the constant values 90 or 270 for an Arm64 SVE vector instruction + * This specifically encode the field 'rot' at bit location '16'. + */ - default: - assert(!"Invalid instruction"); - break; - } +/*static*/ emitter::code_t emitter::insEncodeSveImm90_or_270_rot(ssize_t imm) +{ + assert(emitIsValidEncodedRotationImm90_or_270(imm)); + return (code_t)(imm << 16); +} - switch (ins) - { - case INS_sve_ld2b: - case INS_sve_ld3b: - case INS_sve_ld4b: - assert(opt == INS_OPTS_SCALABLE_B); - break; +/***************************************************************************** + * + * Returns the encoding to select the constant values 0, 90, 180 or 270 for an Arm64 SVE vector instruction + * This specifically encode the field 'rot' at bit locations '14-13'. + */ - case INS_sve_ld2h: - case INS_sve_ld3h: - case INS_sve_ld4h: - assert(opt == INS_OPTS_SCALABLE_H); - break; +/*static*/ emitter::code_t emitter::insEncodeSveImm0_to_270_rot(ssize_t imm) +{ + assert(emitIsValidEncodedRotationImm0_to_270(imm)); + return (code_t)(imm << 13); +} - case INS_sve_ld2w: - case INS_sve_ld3w: - case INS_sve_ld4w: - assert(opt == INS_OPTS_SCALABLE_S); - break; +/***************************************************************************** + * + * Returns the encoding to select the constant float values 0, 0.5, 1.0 or 2.0 for an Arm64 SVE vector instruction + * This specifically encode the field 'i1' at bit location '5'. + */ - case INS_sve_ld2d: - case INS_sve_ld3d: - case INS_sve_ld4d: - assert(opt == INS_OPTS_SCALABLE_D); - break; +/*static*/ emitter::code_t emitter::insEncodeSveSmallFloatImm(ssize_t imm) +{ + assert(emitIsValidEncodedSmallFloatImm(imm)); + return (code_t)(imm << 5); +} - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG +/***************************************************************************** + * + * Returns the register list size for the given SVE instruction. + */ - fmt = IF_SVE_IS_3A; - break; +/*static*/ int emitter::insGetSveReg1ListSize(instruction ins) +{ + switch (ins) + { + case INS_sve_ld1d: + case INS_sve_ld1w: + case INS_sve_ld1sw: + case INS_sve_ld1sb: + case INS_sve_ld1b: + case INS_sve_ld1sh: + case INS_sve_ld1h: + case INS_sve_ldnf1d: + case INS_sve_ldnf1sw: + case INS_sve_ldnf1sh: + case INS_sve_ldnf1w: + case INS_sve_ldnf1h: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1b: + case INS_sve_ldnt1b: + case INS_sve_ldnt1d: + case INS_sve_ldnt1h: + case INS_sve_ldnt1w: + case INS_sve_ld1rob: + case INS_sve_ld1rod: + case INS_sve_ld1roh: + case INS_sve_ld1row: + case INS_sve_ld1rqb: + case INS_sve_ld1rqd: + case INS_sve_ld1rqh: + case INS_sve_ld1rqw: + case INS_sve_stnt1b: + case INS_sve_stnt1d: + case INS_sve_stnt1h: + case INS_sve_stnt1w: + case INS_sve_st1d: + case INS_sve_st1w: + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + case INS_sve_ldff1h: + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + case INS_sve_st1b: + case INS_sve_st1h: + case INS_sve_ldff1sb: + case INS_sve_ldff1b: + case INS_sve_ldnt1sb: + case INS_sve_ldnt1sh: + case INS_sve_ld1rd: + case INS_sve_ld1rsw: + case INS_sve_ld1rh: + case INS_sve_ld1rsb: + case INS_sve_ld1rsh: + case INS_sve_ld1rw: + case INS_sve_ld1q: + case INS_sve_ldnt1sw: + case INS_sve_st1q: + case INS_sve_ld1rb: + return 1; + case INS_sve_ld2b: + case INS_sve_ld2h: + case INS_sve_ld2w: + case INS_sve_ld2d: + case INS_sve_ld2q: + case INS_sve_splice: // SVE_CV_3A + case INS_sve_st2b: + case INS_sve_st2h: + case INS_sve_st2w: + case INS_sve_st2d: case INS_sve_st2q: + case INS_sve_whilege: // SVE_DX_3A + case INS_sve_whilegt: // SVE_DX_3A + case INS_sve_whilehi: // SVE_DX_3A + case INS_sve_whilehs: // SVE_DX_3A + case INS_sve_whilele: // SVE_DX_3A + case INS_sve_whilels: // SVE_DX_3A + case INS_sve_whilelt: // SVE_DX_3A + case INS_sve_pext: // SVE_DW_2B + return 2; + + case INS_sve_ld3b: + case INS_sve_ld3h: + case INS_sve_ld3w: + case INS_sve_ld3d: + case INS_sve_ld3q: + case INS_sve_st3b: + case INS_sve_st3h: + case INS_sve_st3w: + case INS_sve_st3d: case INS_sve_st3q: - case INS_sve_st4q: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_Q); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); + return 3; -#ifdef DEBUG - switch (ins) - { - case INS_sve_st2q: - assert((isValidSimm_MultipleOf<4, 2>(imm))); - break; + case INS_sve_ld4b: + case INS_sve_ld4h: + case INS_sve_ld4w: + case INS_sve_ld4d: + case INS_sve_ld4q: + case INS_sve_st4b: + case INS_sve_st4h: + case INS_sve_st4w: + case INS_sve_st4d: + case INS_sve_st4q: + return 4; - case INS_sve_st3q: - assert((isValidSimm_MultipleOf<4, 3>(imm))); - break; + default: + assert(!"Unexpected instruction"); + return 1; + } +} - case INS_sve_st4q: - assert((isValidSimm_MultipleOf<4, 4>(imm))); - break; +/***************************************************************************** + * + * Returns the predicate type for the given SVE format. + */ - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG +/*static*/ emitter::PredicateType emitter::insGetPredicateType(insFormat fmt, int regpos /* =0 */) +{ + switch (fmt) + { + case IF_SVE_BV_2A: + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + case IF_SVE_HW_4A_B: + case IF_SVE_HW_4A_C: + case IF_SVE_HW_4B: + case IF_SVE_HW_4B_D: + case IF_SVE_HX_3A_E: + case IF_SVE_IJ_3A_D: + case IF_SVE_IJ_3A_E: + case IF_SVE_IJ_3A_F: + case IF_SVE_IK_4A_G: + case IF_SVE_IJ_3A_G: + case IF_SVE_IK_4A_I: + case IF_SVE_IH_3A_F: + case IF_SVE_II_4A_H: + case IF_SVE_IH_3A: + case IF_SVE_IH_3A_A: + case IF_SVE_II_4A: + case IF_SVE_II_4A_B: + case IF_SVE_IU_4A: + case IF_SVE_IU_4A_C: + case IF_SVE_IU_4B: + case IF_SVE_IU_4B_D: + case IF_SVE_IV_3A: + case IF_SVE_IG_4A_F: + case IF_SVE_IG_4A_G: + case IF_SVE_IJ_3A: + case IF_SVE_IK_4A: + case IF_SVE_IK_4A_F: + case IF_SVE_IK_4A_H: + case IF_SVE_IU_4A_A: + case IF_SVE_IU_4B_B: + case IF_SVE_HX_3A_B: + case IF_SVE_IG_4A: + case IF_SVE_IG_4A_D: + case IF_SVE_IG_4A_E: + case IF_SVE_IF_4A: + case IF_SVE_IF_4A_A: + case IF_SVE_IM_3A: + case IF_SVE_IN_4A: + case IF_SVE_IX_4A: + case IF_SVE_IO_3A: + case IF_SVE_IP_4A: + case IF_SVE_IQ_3A: + case IF_SVE_IR_4A: + case IF_SVE_IS_3A: + case IF_SVE_IT_4A: + case IF_SVE_GI_4A: + case IF_SVE_IC_3A_C: + case IF_SVE_IC_3A: + case IF_SVE_IC_3A_B: + case IF_SVE_IC_3A_A: + case IF_SVE_IL_3A_C: + case IF_SVE_IL_3A: + case IF_SVE_IL_3A_B: + case IF_SVE_IL_3A_A: + case IF_SVE_IW_4A: + return PREDICATE_ZERO; + + case IF_SVE_BV_2A_J: + case IF_SVE_CP_3A: + case IF_SVE_CQ_3A: + case IF_SVE_AM_2A: + case IF_SVE_AN_3A: + case IF_SVE_AO_3A: + case IF_SVE_HL_3A: + case IF_SVE_HM_2A: + case IF_SVE_AA_3A: + case IF_SVE_BU_2A: + case IF_SVE_BV_2B: + case IF_SVE_HS_3A: + case IF_SVE_HP_3A: + case IF_SVE_HP_3B: + case IF_SVE_AR_4A: + case IF_SVE_BV_2A_A: + case IF_SVE_AB_3A: + case IF_SVE_ET_3A: + case IF_SVE_HU_4A: + case IF_SVE_HL_3B: + case IF_SVE_AD_3A: + case IF_SVE_AB_3B: + case IF_SVE_AE_3A: + case IF_SVE_EU_3A: + case IF_SVE_GT_4A: + case IF_SVE_AP_3A: + case IF_SVE_HO_3A: + case IF_SVE_HO_3B: + case IF_SVE_HO_3C: + case IF_SVE_GQ_3A: + case IF_SVE_HU_4B: + case IF_SVE_AQ_3A: + case IF_SVE_CU_3A: + case IF_SVE_AC_3A: + case IF_SVE_ER_3A: + case IF_SVE_GR_3A: + case IF_SVE_ES_3A: + case IF_SVE_HR_3A: + case IF_SVE_EP_3A: + case IF_SVE_GP_3A: + case IF_SVE_EQ_3A: + case IF_SVE_HQ_3A: + case IF_SVE_AS_4A: + case IF_SVE_CT_3A: + case IF_SVE_HV_4A: + return PREDICATE_MERGE; + + case IF_SVE_CZ_4A_A: + case IF_SVE_CZ_4A_L: + case IF_SVE_CE_2A: + case IF_SVE_CE_2B: + case IF_SVE_CE_2C: + case IF_SVE_CE_2D: + case IF_SVE_CF_2A: + case IF_SVE_CF_2B: + case IF_SVE_CF_2C: + case IF_SVE_CF_2D: + case IF_SVE_CI_3A: + case IF_SVE_CJ_2A: + case IF_SVE_DE_1A: + case IF_SVE_DH_1A: + case IF_SVE_DJ_1A: + case IF_SVE_DM_2A: + case IF_SVE_DN_2A: + case IF_SVE_DO_2A: + case IF_SVE_DP_2A: + case IF_SVE_DR_1A: + case IF_SVE_DT_3A: + case IF_SVE_DU_3A: + case IF_SVE_CK_2A: + return PREDICATE_SIZED; + + case IF_SVE_DB_3A: + // Second register could be ZERO or MERGE so handled at source. + assert(regpos != 2); + return PREDICATE_SIZED; + + case IF_SVE_DL_2A: + case IF_SVE_DY_3A: + case IF_SVE_DZ_1A: + return PREDICATE_N_SIZED; + + // This is a special case as the second register could be ZERO or MERGE. + // / + // Therefore, by default return NONE due to ambiguity. + case IF_SVE_AH_3A: + // TODO: Handle these cases. + assert(false); + break; + + case IF_SVE_JD_4B: + case IF_SVE_JD_4C: + case IF_SVE_JI_3A_A: + case IF_SVE_JJ_4A: + case IF_SVE_JJ_4A_B: + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + case IF_SVE_JJ_4B: + case IF_SVE_JJ_4B_E: + case IF_SVE_JN_3B: + case IF_SVE_JN_3C: + case IF_SVE_JD_4A: + case IF_SVE_JN_3A: + case IF_SVE_JD_4C_A: + case IF_SVE_JJ_4B_C: + case IF_SVE_JL_3A: + case IF_SVE_JN_3C_D: + case IF_SVE_HY_3A: + case IF_SVE_HY_3A_A: + case IF_SVE_HY_3B: + case IF_SVE_HZ_2A_B: + case IF_SVE_IA_2A: + case IF_SVE_IB_3A: + case IF_SVE_JK_4A: + case IF_SVE_JK_4A_B: + case IF_SVE_JK_4B: + case IF_SVE_IZ_4A: + case IF_SVE_IZ_4A_A: + case IF_SVE_JB_4A: + case IF_SVE_JM_3A: + case IF_SVE_CM_3A: + case IF_SVE_CN_3A: + case IF_SVE_CO_3A: + case IF_SVE_JA_4A: + case IF_SVE_CR_3A: + case IF_SVE_CS_3A: + case IF_SVE_CV_3A: + case IF_SVE_CV_3B: + case IF_SVE_DW_2A: // [] + case IF_SVE_DW_2B: // [] + case IF_SVE_JC_4A: + case IF_SVE_JO_3A: + case IF_SVE_JE_3A: + case IF_SVE_JF_4A: + case IF_SVE_AK_3A: + case IF_SVE_HE_3A: + case IF_SVE_AF_3A: + case IF_SVE_AG_3A: + case IF_SVE_AI_3A: + case IF_SVE_AJ_3A: + case IF_SVE_AL_3A: + case IF_SVE_CL_3A: + case IF_SVE_GS_3A: + case IF_SVE_HJ_3A: + case IF_SVE_IY_4A: + return PREDICATE_NONE; + + case IF_SVE_CX_4A: + case IF_SVE_CX_4A_A: + case IF_SVE_CY_3A: + case IF_SVE_CY_3B: + case IF_SVE_GE_4A: + case IF_SVE_HT_4A: + assert((regpos == 1) || (regpos == 2)); + return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); + + case IF_SVE_CZ_4A: + case IF_SVE_DA_4A: + case IF_SVE_DB_3B: + case IF_SVE_DC_3A: + assert((regpos >= 1) && (regpos <= 4)); + return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); + + case IF_SVE_CZ_4A_K: + assert((regpos >= 1) && (regpos <= 3)); + return (regpos == 2 ? PREDICATE_MERGE : PREDICATE_SIZED); + + case IF_SVE_DD_2A: + case IF_SVE_DF_2A: + assert((regpos >= 1) && (regpos <= 3)); + return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); + + case IF_SVE_DG_2A: + return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); + + case IF_SVE_DI_2A: + return (regpos == 1 ? PREDICATE_NONE : PREDICATE_SIZED); + + case IF_SVE_DK_3A: + assert((regpos == 2) || (regpos == 3)); + return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); + + case IF_SVE_HI_3A: + assert((regpos == 1) || (regpos == 2)); + return ((regpos == 2) ? PREDICATE_ZERO : PREDICATE_SIZED); + + case IF_SVE_DV_4A: + assert((regpos >= 1) && (regpos <= 3)); + return ((regpos == 3) ? PREDICATE_SIZED : PREDICATE_NONE); + + case IF_SVE_ID_2A: + case IF_SVE_JG_2A: + return PREDICATE_NONE; - fmt = IF_SVE_JE_3A; + default: break; + } - case INS_sve_stnt1b: - case INS_sve_stnt1h: - case INS_sve_stnt1w: - case INS_sve_stnt1d: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + assert(!"Unexpected instruction format"); + return PREDICATE_NONE; +} -#ifdef DEBUG +/***************************************************************************** + * + * Returns true if the SVE instruction has a LSL addr. + * This is for formats that have [, , LSL #N], [{, , LSL #N}] + */ +/*static*/ bool emitter::insSveIsLslN(instruction ins, insFormat fmt) +{ + switch (fmt) + { + case IF_SVE_JD_4A: switch (ins) { - case INS_sve_stnt1b: - assert(opt == INS_OPTS_SCALABLE_B); - break; - - case INS_sve_stnt1h: - assert(opt == INS_OPTS_SCALABLE_H); - break; - - case INS_sve_stnt1w: - assert(opt == INS_OPTS_SCALABLE_S); - break; - - case INS_sve_stnt1d: - assert(opt == INS_OPTS_SCALABLE_D); - break; + case INS_sve_st1h: + return true; default: - assert(!"Invalid instruction"); break; } -#endif // DEBUG - - fmt = IF_SVE_JM_3A; break; - case INS_sve_st1w: - case INS_sve_st1d: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - - if (isGeneralRegister(reg3)) + case IF_SVE_JD_4B: + switch (ins) { - assert(isValidSimm<4>(imm)); + case INS_sve_st1w: + return true; - if (opt == INS_OPTS_SCALABLE_Q && (ins == INS_sve_st1d)) - { - fmt = IF_SVE_JN_3C_D; - } - else - { - if ((ins == INS_sve_st1w) && insOptsScalableWords(opt)) - { - fmt = IF_SVE_JN_3B; - } - else - { -#if DEBUG - if (ins == INS_sve_st1w) - { - assert(opt == INS_OPTS_SCALABLE_Q); - } - else - { - assert(opt == INS_OPTS_SCALABLE_D); - } -#endif // DEBUG - fmt = IF_SVE_JN_3C; - } - } - } - else - { - assert(isVectorRegister(reg3)); - if ((ins == INS_sve_st1w) && insOptsScalableWords(opt)) - { - assert((isValidUimm_MultipleOf<5, 4>(imm))); - fmt = IF_SVE_JI_3A_A; - } - else - { - assert(ins == INS_sve_st1d); - assert((isValidUimm_MultipleOf<5, 8>(imm))); - fmt = IF_SVE_JL_3A; - } + default: + break; } break; - case INS_sve_st2b: - case INS_sve_st3b: - case INS_sve_st4b: - case INS_sve_st2h: - case INS_sve_st3h: - case INS_sve_st4h: - case INS_sve_st2w: - case INS_sve_st3w: - case INS_sve_st4w: - case INS_sve_st2d: - case INS_sve_st3d: - case INS_sve_st4d: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - -#ifdef DEBUG + case IF_SVE_HW_4B: switch (ins) { - case INS_sve_st2b: - case INS_sve_st2h: - case INS_sve_st2w: - case INS_sve_st2d: - assert((isValidSimm_MultipleOf<4, 2>(imm))); - break; + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + case INS_sve_ld1w: + case INS_sve_ldff1w: + return true; - case INS_sve_st3b: - case INS_sve_st3h: - case INS_sve_st3w: - case INS_sve_st3d: - assert((isValidSimm_MultipleOf<4, 3>(imm))); + default: break; + } + break; - case INS_sve_st4b: - case INS_sve_st4h: - case INS_sve_st4w: - case INS_sve_st4d: - assert((isValidSimm_MultipleOf<4, 4>(imm))); - break; + case IF_SVE_IG_4A: + switch (ins) + { + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + return true; default: - assert(!"Invalid instruction"); break; } + break; + case IF_SVE_IG_4A_F: switch (ins) { - case INS_sve_st2b: - case INS_sve_st3b: - case INS_sve_st4b: - assert(opt == INS_OPTS_SCALABLE_B); - break; - - case INS_sve_st2h: - case INS_sve_st3h: - case INS_sve_st4h: - assert(opt == INS_OPTS_SCALABLE_H); - break; + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + return true; - case INS_sve_st2w: - case INS_sve_st3w: - case INS_sve_st4w: - assert(opt == INS_OPTS_SCALABLE_S); + default: break; + } + break; - case INS_sve_st2d: - case INS_sve_st3d: - case INS_sve_st4d: - assert(opt == INS_OPTS_SCALABLE_D); - break; + case IF_SVE_IG_4A_G: + switch (ins) + { + case INS_sve_ldff1h: + return true; default: - assert(!"Invalid instruction"); break; } -#endif // DEBUG - - fmt = IF_SVE_JO_3A; break; - case INS_sve_st1b: - case INS_sve_st1h: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - - if (isGeneralRegister(reg3)) - { - assert(isValidSimm<4>(imm)); - // st1h is reserved for scalable B - assert((ins == INS_sve_st1h) ? insOptsScalableAtLeastHalf(opt) : insOptsScalableStandard(opt)); - fmt = IF_SVE_JN_3A; - } - else + case IF_SVE_II_4A: + case IF_SVE_II_4A_B: + switch (ins) { - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg3)); - -#ifdef DEBUG - switch (ins) - { - case INS_sve_st1b: - assert(isValidUimm<5>(imm)); - break; - - case INS_sve_st1h: - assert((isValidUimm_MultipleOf<5, 2>(imm))); - break; - - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG + case INS_sve_ld1d: + return true; - fmt = IF_SVE_JI_3A_A; + default: + break; } break; - case INS_sve_fmla: - case INS_sve_fmls: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - - if (opt == INS_OPTS_SCALABLE_S) - { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_GU_3A; - } - else + case IF_SVE_II_4A_H: + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_GU_3B; + case INS_sve_ld1w: + return true; + + default: + break; } break; - case INS_sve_bfmla: - case INS_sve_bfmls: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmm - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<3>(imm)); // i ii - fmt = IF_SVE_GU_3C; - break; + case IF_SVE_IK_4A: + switch (ins) + { + case INS_sve_ld1sw: + return true; - case INS_sve_fmul: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm + default: + break; + } + break; - if (opt == INS_OPTS_SCALABLE_S) + case IF_SVE_IK_4A_G: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_GX_3A; + case INS_sve_ld1sh: + return true; + + default: + break; } - else + break; + + case IF_SVE_IK_4A_I: + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_GX_3B; + case INS_sve_ld1h: + return true; + + default: + break; } break; - case INS_sve_bfmul: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmm - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<3>(imm)); // i ii - fmt = IF_SVE_GX_3C; - break; + case IF_SVE_IN_4A: + switch (ins) + { + case INS_sve_ldnt1d: + case INS_sve_ldnt1h: + case INS_sve_ldnt1w: + return true; - case INS_sve_fdot: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmm - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<2>(imm)); // ii + default: + break; + } + break; - if (opt == INS_OPTS_SCALABLE_B) + case IF_SVE_IP_4A: + switch (ins) { - unreached(); // TODO-SVE: Not yet supported. - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_GY_3B_D; + case INS_sve_ld1roh: + case INS_sve_ld1row: + case INS_sve_ld1rod: + case INS_sve_ld1rqh: + case INS_sve_ld1rqw: + case INS_sve_ld1rqd: + return true; + + default: + break; } - else if (opt == INS_OPTS_SCALABLE_H) + break; + + case IF_SVE_IR_4A: + switch (ins) { - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_GY_3B; + case INS_sve_ld2q: + case INS_sve_ld3q: + case INS_sve_ld4q: + return true; + + default: + break; } - else + break; + + case IF_SVE_IT_4A: + switch (ins) { - unreached(); // TODO-SVE: Not yet supported. - assert(insOptsNone(opt)); - assert(isValidUimm<3>(imm)); // i ii + case INS_sve_ld2h: + case INS_sve_ld2w: + case INS_sve_ld2d: + case INS_sve_ld3h: + case INS_sve_ld3w: + case INS_sve_ld3d: + case INS_sve_ld4h: + case INS_sve_ld4w: + case INS_sve_ld4d: + return true; - // Simplify emitDispInsHelp logic by setting insOpt - opt = INS_OPTS_SCALABLE_B; - fmt = IF_SVE_GY_3A; + default: + break; } break; - case INS_sve_bfdot: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmm - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_GY_3B; - break; + case IF_SVE_IU_4B: + switch (ins) + { + case INS_sve_ld1sw: + case INS_sve_ldff1sw: + case INS_sve_ld1d: + case INS_sve_ldff1d: + return true; - case INS_sve_mla: - case INS_sve_mls: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm + default: + break; + } + break; - if (opt == INS_OPTS_SCALABLE_H) + case IF_SVE_JB_4A: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // i ii - fmt = IF_SVE_FF_3A; + case INS_sve_stnt1h: + case INS_sve_stnt1w: + case INS_sve_stnt1d: + return true; + + default: + break; } - else if (opt == INS_OPTS_SCALABLE_S) + break; + + case IF_SVE_JC_4A: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_FF_3B; + case INS_sve_st2h: + case INS_sve_st2w: + case INS_sve_st2d: + case INS_sve_st3h: + case INS_sve_st3w: + case INS_sve_st3d: + case INS_sve_st4h: + case INS_sve_st4w: + case INS_sve_st4d: + return true; + + default: + break; } - else + break; + + case IF_SVE_JD_4C: + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_FF_3C; + case INS_sve_st1w: + case INS_sve_st1d: + return true; + + default: + break; } break; - case INS_sve_smullb: - case INS_sve_smullt: - case INS_sve_umullb: - case INS_sve_umullt: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - - if (opt == INS_OPTS_SCALABLE_H) + case IF_SVE_JD_4C_A: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i - fmt = IF_SVE_FE_3A; + case INS_sve_st1d: + return true; + + default: + break; } - else + break; + + case IF_SVE_JF_4A: + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<2>(imm)); // i i - fmt = IF_SVE_FE_3B; + case INS_sve_st2q: + case INS_sve_st3q: + case INS_sve_st4q: + return true; + + default: + break; } break; - case INS_sve_smlalb: - case INS_sve_smlalt: - case INS_sve_umlalb: - case INS_sve_umlalt: - case INS_sve_smlslb: - case INS_sve_smlslt: - case INS_sve_umlslb: - case INS_sve_umlslt: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - - if (opt == INS_OPTS_SCALABLE_H) + case IF_SVE_JJ_4B: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i - fmt = IF_SVE_FG_3A; + case INS_sve_st1h: + case INS_sve_st1w: + case INS_sve_st1d: + return true; + + default: + break; } - else + break; + + case IF_SVE_HY_3B: + case IF_SVE_IB_3A: + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<2>(imm)); // i i - fmt = IF_SVE_FG_3B; + case INS_sve_prfh: + case INS_sve_prfw: + case INS_sve_prfd: + return true; + + default: + break; } break; - case INS_sve_sqdmullb: - case INS_sve_sqdmullt: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm + default: + break; + } - if (opt == INS_OPTS_SCALABLE_H) + return false; +} + +/***************************************************************************** + * + * Returns true if the SVE instruction has a addr. + * This is for formats that have [, .T, ], [, .T, #N] + */ +/*static*/ bool emitter::insSveIsModN(instruction ins, insFormat fmt) +{ + switch (fmt) + { + case IF_SVE_JJ_4A: + case IF_SVE_JJ_4A_B: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i - fmt = IF_SVE_FH_3A; + case INS_sve_st1d: + case INS_sve_st1h: + case INS_sve_st1w: + return true; + + default: + break; } - else + break; + + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<2>(imm)); // i i - fmt = IF_SVE_FH_3B; + case INS_sve_st1h: + case INS_sve_st1w: + return true; + + default: + break; + } + break; + + case IF_SVE_JK_4A: + case IF_SVE_JK_4A_B: + switch (ins) + { + case INS_sve_st1b: + return true; + + default: + break; } break; - case INS_sve_sqdmulh: - case INS_sve_sqrdmulh: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - - if (opt == INS_OPTS_SCALABLE_H) + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i - fmt = IF_SVE_FI_3A; + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ld1sb: + case INS_sve_ld1sh: + case INS_sve_ld1w: + case INS_sve_ldff1b: + case INS_sve_ldff1h: + case INS_sve_ldff1sb: + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + return true; + + default: + break; } - else if (opt == INS_OPTS_SCALABLE_S) + break; + + case IF_SVE_HW_4A_B: + case IF_SVE_HW_4A_C: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_FI_3B; + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ld1w: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + return true; + + default: + break; } - else + break; + + case IF_SVE_IU_4A: + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_FI_3C; + case INS_sve_ld1d: + case INS_sve_ld1sw: + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + return true; + + default: + break; } break; - case INS_sve_sqdmlalb: - case INS_sve_sqdmlalt: - case INS_sve_sqdmlslb: - case INS_sve_sqdmlslt: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm + case IF_SVE_IU_4A_A: + switch (ins) + { + case INS_sve_ld1sw: + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + return true; - if (opt == INS_OPTS_SCALABLE_H) + default: + break; + } + break; + + case IF_SVE_IU_4A_C: + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i - fmt = IF_SVE_FJ_3A; + case INS_sve_ld1d: + return true; + + default: + break; } - else + break; + + case IF_SVE_HY_3A: + case IF_SVE_HY_3A_A: + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_FJ_3B; + case INS_sve_prfb: + case INS_sve_prfh: + case INS_sve_prfw: + case INS_sve_prfd: + return true; + + default: + break; } break; - case INS_sve_sqrdmlah: - case INS_sve_sqrdmlsh: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm + default: + break; + } - if (opt == INS_OPTS_SCALABLE_H) + return false; +} + +/***************************************************************************** + * + * Returns 0, 1, 2, 3 or 4 depending on the instruction and format. + * This is for formats that have [, .T, ], [, .T, #N], [, , LSL #N], + * [{, , LSL #N}] + */ + +/*static*/ int emitter::insSveGetLslOrModN(instruction ins, insFormat fmt) +{ + switch (fmt) + { + case IF_SVE_JD_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // i ii - fmt = IF_SVE_FK_3A; + case INS_sve_st1h: + return 1; + + default: + break; } - else if (opt == INS_OPTS_SCALABLE_S) + break; + + case IF_SVE_JD_4B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii - fmt = IF_SVE_FK_3B; + case INS_sve_st1w: + return 2; + + default: + break; } - else + break; + + case IF_SVE_HW_4B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_FK_3C; + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + return 1; + + case INS_sve_ld1w: + case INS_sve_ldff1w: + return 2; + + default: + break; } break; - case INS_sve_fcadd: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isScalableVectorSize(size)); - imm = emitEncodeRotationImm90_or_270(imm); - fmt = IF_SVE_GP_3A; - break; + case IF_SVE_JJ_4A: + case IF_SVE_JJ_4A_B: + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + case IF_SVE_JK_4A: + case IF_SVE_JK_4A_B: + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + case IF_SVE_HW_4A_B: + case IF_SVE_HW_4A_C: + case IF_SVE_IU_4A: + case IF_SVE_IU_4A_A: + case IF_SVE_IU_4A_C: + assert(!insSveIsLslN(ins, fmt)); + assert(insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + switch (fmt) + { + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + return 1; - case INS_sve_ld1rd: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 8>(imm))); - fmt = IF_SVE_IC_3A; - break; + default: + break; + } + return 0; - case INS_sve_ld1rsw: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 4>(imm))); - fmt = IF_SVE_IC_3A; - break; + case INS_sve_ld1w: + case INS_sve_ldff1w: + case INS_sve_ld1sw: + case INS_sve_ldff1sw: + switch (fmt) + { + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + case IF_SVE_IU_4A: + return 2; - case INS_sve_ld1rsh: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 2>(imm))); - fmt = IF_SVE_IC_3A_A; - break; + default: + break; + } + return 0; - case INS_sve_ld1rw: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 4>(imm))); - fmt = IF_SVE_IC_3A_A; - break; + case INS_sve_ld1d: + case INS_sve_ldff1d: + switch (fmt) + { + case IF_SVE_IU_4A: + return 3; - case INS_sve_ld1rh: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 2>(imm))); - fmt = IF_SVE_IC_3A_B; - break; + default: + break; + } + return 0; - case INS_sve_ld1rsb: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidUimm<6>(imm)); - fmt = IF_SVE_IC_3A_B; - break; + case INS_sve_st1h: + switch (fmt) + { + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + return 0; - case INS_sve_ld1rb: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isValidUimm<6>(imm)); - fmt = IF_SVE_IC_3A_C; + default: + break; + } + return 1; + + case INS_sve_st1w: + switch (fmt) + { + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + return 0; + + default: + break; + } + return 2; + + case INS_sve_st1d: + if (fmt == IF_SVE_JJ_4A_B) + { + return 0; + } + return 3; + + default: + break; + } + return 0; + + case IF_SVE_IG_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ldff1sw: + return 2; + + case INS_sve_ldff1d: + return 3; + + default: + break; + } break; - case INS_sve_fmlalb: - case INS_sve_fmlalt: - case INS_sve_fmlslb: - case INS_sve_fmlslt: - case INS_sve_bfmlalb: - case INS_sve_bfmlalt: - case INS_sve_bfmlslb: - case INS_sve_bfmlslt: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmm - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<3>(imm)); // ii i - fmt = IF_SVE_GZ_3A; + case IF_SVE_IG_4A_F: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ldff1sh: + return 1; + + case INS_sve_ldff1w: + return 2; + + default: + break; + } break; - case INS_sve_luti2: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + case IF_SVE_IG_4A_G: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ldff1h: + return 1; - if (opt == INS_OPTS_SCALABLE_H) + default: + break; + } + break; + + case IF_SVE_II_4A: + case IF_SVE_II_4A_B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(isValidUimm<3>(imm)); // iii - fmt = IF_SVE_GG_3B; + case INS_sve_ld1d: + return 3; + + default: + break; } - else + break; + + case IF_SVE_II_4A_H: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isValidUimm<2>(imm)); // ii i - fmt = IF_SVE_GG_3A; + case INS_sve_ld1w: + return 2; + + default: + break; } - unreached(); break; - case INS_sve_luti4: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + case IF_SVE_IK_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1sw: + return 2; - if (opt == INS_OPTS_SCALABLE_H) + default: + break; + } + break; + + case IF_SVE_IK_4A_G: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(isValidUimm<2>(imm)); + case INS_sve_ld1sh: + return 1; - if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) - { - fmt = IF_SVE_GH_3B; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_GH_3B_B; - } + default: + break; } - else + break; + + case IF_SVE_IK_4A_I: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_GH_3A; + case INS_sve_ld1h: + return 1; + + default: + break; } - unreached(); break; - default: - unreached(); + case IF_SVE_IN_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ldnt1h: + return 1; + case INS_sve_ldnt1w: + return 2; + case INS_sve_ldnt1d: + return 3; + + default: + break; + } break; - } // end switch (ins) - assert(fmt != IF_NONE); + case IF_SVE_IP_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1roh: + case INS_sve_ld1rqh: + return 1; - instrDesc* id = emitNewInstrCns(attr, imm); + case INS_sve_ld1row: + case INS_sve_ld1rqw: + return 2; + case INS_sve_ld1rod: + case INS_sve_ld1rqd: + return 3; - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); + default: + break; + } + break; - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); + case IF_SVE_IR_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld2q: + case INS_sve_ld3q: + case INS_sve_ld4q: + return 4; - dispIns(id); - appendToCurIG(id); -} + default: + break; + } + break; -/***************************************************************************** - * - * Add a SVE instruction referencing three registers and two constants. - */ + case IF_SVE_IT_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld2h: + case INS_sve_ld3h: + case INS_sve_ld4h: + return 1; -void emitter::emitInsSve_R_R_R_I_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm1, - ssize_t imm2, - insOpts opt) -{ - insFormat fmt = IF_NONE; - ssize_t imm; + case INS_sve_ld2w: + case INS_sve_ld3w: + case INS_sve_ld4w: + return 2; - switch (ins) - { - case INS_sve_cdot: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - assert(isValidRot(imm2)); // rr - // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) - imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); + case INS_sve_ld2d: + case INS_sve_ld3d: + case INS_sve_ld4d: + return 3; - if (opt == INS_OPTS_SCALABLE_B) + default: + break; + } + break; + + case IF_SVE_IU_4B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(isValidUimm<2>(imm1)); // ii - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - fmt = IF_SVE_FA_3A; + case INS_sve_ld1sw: + case INS_sve_ldff1sw: + return 2; + + case INS_sve_ld1d: + case INS_sve_ldff1d: + return 3; + + default: + break; } - else + break; + + case IF_SVE_JB_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_H); - assert(isValidUimm<1>(imm1)); // i - fmt = IF_SVE_FA_3B; + case INS_sve_stnt1h: + return 1; + + case INS_sve_stnt1w: + return 2; + + case INS_sve_stnt1d: + return 3; + + default: + break; } break; - case INS_sve_cmla: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - assert(isValidRot(imm2)); // rr - // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) - imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); + case IF_SVE_JC_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st2h: + case INS_sve_st3h: + case INS_sve_st4h: + return 1; - if (opt == INS_OPTS_SCALABLE_H) + case INS_sve_st2w: + case INS_sve_st3w: + case INS_sve_st4w: + return 2; + + case INS_sve_st2d: + case INS_sve_st3d: + case INS_sve_st4d: + return 3; + + default: + break; + } + break; + + case IF_SVE_JD_4C: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(isValidUimm<2>(imm1)); // ii - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - fmt = IF_SVE_FB_3A; + case INS_sve_st1w: + return 2; + + case INS_sve_st1d: + return 3; + + default: + break; } - else + break; + + case IF_SVE_JD_4C_A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<1>(imm1)); // i - fmt = IF_SVE_FB_3B; + case INS_sve_st1d: + return 3; + + default: + break; } break; - case INS_sve_sqrdcmlah: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - assert(isValidRot(imm2)); // rr - // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) - imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); + case IF_SVE_JF_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st2q: + case INS_sve_st3q: + case INS_sve_st4q: + return 4; - if (opt == INS_OPTS_SCALABLE_H) + default: + break; + } + break; + + case IF_SVE_JJ_4B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) { - assert(isValidUimm<2>(imm1)); // ii - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - fmt = IF_SVE_FC_3A; + case INS_sve_st1h: + return 1; + + case INS_sve_st1w: + return 2; + + case INS_sve_st1d: + return 3; + + default: + break; } - else + break; + + case IF_SVE_HY_3A: + case IF_SVE_HY_3A_A: + assert(!insSveIsLslN(ins, fmt)); + assert(insSveIsModN(ins, fmt)); + switch (ins) { - assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<1>(imm1)); // i - fmt = IF_SVE_FC_3B; + case INS_sve_prfb: + return 0; + + case INS_sve_prfh: + return 1; + + case INS_sve_prfw: + return 2; + + case INS_sve_prfd: + return 3; + + default: + break; } break; - case INS_sve_fcmla: - assert(opt == INS_OPTS_SCALABLE_S); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isLowVectorRegister(reg3)); // mmmm - assert(isValidUimm<1>(imm1)); // i - assert(isValidRot(imm2)); // rr + case IF_SVE_HY_3B: + case IF_SVE_IB_3A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_prfh: + return 1; + + case INS_sve_prfw: + return 2; + + case INS_sve_prfd: + return 3; - // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) - imm = (imm1 << 2) | emitEncodeRotationImm0_to_270(imm2); - fmt = IF_SVE_GV_3A; + default: + break; + } break; default: - unreached(); break; } - assert(fmt != IF_NONE); + assert(!"Unexpected instruction format"); + return 0; +} - instrDesc* id = emitNewInstrCns(attr, imm); - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); +/***************************************************************************** + * + * Returns true if the specified instruction can encode the 'dtype' field. + */ - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); +/*static*/ bool emitter::canEncodeSveElemsize_dtype(instruction ins) +{ + switch (ins) + { + case INS_sve_ld1w: + case INS_sve_ld1sb: + case INS_sve_ld1b: + case INS_sve_ld1sh: + case INS_sve_ld1h: + case INS_sve_ldnf1sh: + case INS_sve_ldnf1w: + case INS_sve_ldnf1h: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1b: + case INS_sve_ldff1b: + case INS_sve_ldff1sb: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + return true; - dispIns(id); - appendToCurIG(id); + default: + return false; + } } /***************************************************************************** * - * Add a SVE instruction referencing four registers. - * Do not call this directly. Use 'emitIns_R_R_R_R' instead. + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * for the 'dtype' field. */ -void emitter::emitInsSve_R_R_R_R(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - regNumber reg4, - insOpts opt /* = INS_OPTS_NONE*/, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(instruction ins, emitAttr size, code_t code) { - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - switch (ins) + assert(canEncodeSveElemsize_dtype(ins)); + assert(ins != INS_sve_ld1w); + switch (size) { - case INS_sve_sel: - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + case EA_1BYTE: + switch (ins) { - if (reg1 == reg4) - { - // mov is a preferred alias for sel - return emitInsSve_R_R_R(INS_sve_mov, attr, reg1, reg2, reg3, opt, - INS_SCALABLE_OPTS_PREDICATE_MERGE); - } + case INS_sve_ld1b: + case INS_sve_ldnf1b: + case INS_sve_ldff1b: + return code; // By default, the instruction already encodes 8-bit. - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isPredicateRegister(reg2)); // VVVV - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - fmt = IF_SVE_CW_4A; - } - else - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // dddd - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // nnnn - assert(isPredicateRegister(reg4)); // mmmm - fmt = IF_SVE_CZ_4A; + default: + assert(!"Invalid instruction for encoding dtype."); } - break; + return code; - case INS_sve_cmpeq: - case INS_sve_cmpgt: - case INS_sve_cmpge: - case INS_sve_cmphi: - case INS_sve_cmphs: - case INS_sve_cmpne: - case INS_sve_cmple: - case INS_sve_cmplo: - case INS_sve_cmpls: - case INS_sve_cmplt: - assert(isPredicateRegister(reg1)); // DDDD - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - assert(isScalableVectorSize(attr)); // xx - if (sopt == INS_SCALABLE_OPTS_WIDE) + case EA_2BYTE: + switch (ins) { - assert(insOptsScalableWide(opt)); - fmt = IF_SVE_CX_4A_A; + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ldnf1b: + case INS_sve_ldnf1h: + case INS_sve_ldff1b: + case INS_sve_ldff1h: + return code | (1 << 21); // Set bit '21' to 1. + + case INS_sve_ld1sb: + case INS_sve_ldnf1sb: + case INS_sve_ldff1sb: + return code | (1 << 22); // Set bit '22' to 1. + + default: + assert(!"Invalid instruction for encoding dtype."); } - else + return code; + + case EA_4BYTE: + switch (ins) { - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_CX_4A; + case INS_sve_ldnf1w: + case INS_sve_ldff1w: + return code; // By default, the instruction already encodes 32-bit. + + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ldnf1b: + case INS_sve_ldnf1h: + case INS_sve_ldff1b: + case INS_sve_ldff1h: + return code | (1 << 22); // Set bit '22' to 1. + + case INS_sve_ld1sb: + case INS_sve_ld1sh: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1sh: + case INS_sve_ldff1sb: + case INS_sve_ldff1sh: + return code | (1 << 21); // Set bit '21' to 1. + + default: + assert(!"Invalid instruction for encoding dtype."); } - break; + return code; - case INS_sve_and: - case INS_sve_orr: - case INS_sve_eor: - case INS_sve_ands: - case INS_sve_bic: - case INS_sve_orn: - case INS_sve_bics: - case INS_sve_eors: - case INS_sve_nor: - case INS_sve_nand: - case INS_sve_orrs: - case INS_sve_orns: - case INS_sve_nors: - case INS_sve_nands: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // dddd - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // nnnn - assert(isPredicateRegister(reg4)); // mmmm - fmt = IF_SVE_CZ_4A; - break; + case EA_8BYTE: + switch (ins) + { + case INS_sve_ldnf1w: + case INS_sve_ldff1w: + return code | (1 << 21); // Set bit '21' to 1. Set bit '15' to 1. + + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ldnf1b: + case INS_sve_ldnf1h: + case INS_sve_ldff1b: + case INS_sve_ldff1h: + return (code | (1 << 22)) | (1 << 21); // Set bit '22' and '21' to 1. + + case INS_sve_ld1sb: + case INS_sve_ld1sh: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1sh: + case INS_sve_ldff1sb: + case INS_sve_ldff1sh: + return code; // By default, the instruction already encodes 64-bit. - case INS_sve_brkpa: - case INS_sve_brkpb: - case INS_sve_brkpas: - case INS_sve_brkpbs: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // dddd - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // nnnn - assert(isPredicateRegister(reg4)); // mmmm - fmt = IF_SVE_DA_4A; - break; + default: + assert(!"Invalid instruction for encoding dtype."); + } + return code; - case INS_sve_fcmeq: - case INS_sve_fcmge: - case INS_sve_facge: - case INS_sve_fcmgt: - case INS_sve_facgt: - case INS_sve_fcmlt: - case INS_sve_fcmle: - case INS_sve_fcmne: - case INS_sve_fcmuo: - case INS_sve_facle: - case INS_sve_faclt: - assert(insOptsScalableFloat(opt)); - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - assert(isPredicateRegister(reg1)); // DDDD - assert(isLowPredicateRegister(reg2)); // ggg - assert(isScalableVectorSize(attr)); // xx - fmt = IF_SVE_HT_4A; - break; + default: + assert(!"Invalid size for encoding dtype."); + } - case INS_sve_match: - case INS_sve_nmatch: - assert(insOptsScalableAtMaxHalf(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - assert(isScalableVectorSize(attr)); // xx - fmt = IF_SVE_GE_4A; - break; + return code; +} - case INS_sve_mla: - case INS_sve_mls: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - assert(isScalableVectorSize(size)); - fmt = IF_SVE_AR_4A; - break; +/***************************************************************************** + * + * Returns the encoding to select the 4/8/16 byte elemsize for the Arm64 Sve vector instruction 'ld1w' + * for the 'dtype' field. + */ - case INS_sve_histcnt: - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_GI_4A; - break; +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype_ld1w(instruction ins, + insFormat fmt, + emitAttr size, + code_t code) +{ + assert(canEncodeSveElemsize_dtype(ins)); + assert(ins == INS_sve_ld1w); + switch (size) + { + case EA_4BYTE: + switch (fmt) + { + case IF_SVE_IH_3A_F: + // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for S. + return (code | (1 << 15)) | (1 << 22); // Set bit '22' and '15' to 1. - case INS_sve_fmla: - case INS_sve_fmls: - case INS_sve_fnmla: - case INS_sve_fnmls: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_HU_4A; - break; + case IF_SVE_II_4A_H: + // Note: Bit '14' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for S. + return (code | (1 << 14)) | (1 << 22); // Set bit '22' and '14' to 1. - case INS_sve_mad: - case INS_sve_msb: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // mmmmm - assert(isVectorRegister(reg4)); // aaaaa - assert(isScalableVectorSize(size)); - fmt = IF_SVE_AS_4A; + default: + break; + } break; - - case INS_sve_st1b: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isScalableVectorSize(size)); - assert(insScalableOptsNone(sopt)); - - if (insOptsScalableStandard(opt)) - { - if (isGeneralRegister(reg4)) - { - fmt = IF_SVE_JD_4A; - } - else - { - assert(isVectorRegister(reg4)); - fmt = IF_SVE_JK_4B; - } + + case EA_8BYTE: + switch (fmt) + { + case IF_SVE_IH_3A_F: + // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for D. + return ((code | (1 << 15)) | (1 << 22)) | (1 << 21); // Set bit '22', '21' and '15' to 1. + + case IF_SVE_II_4A_H: + // Note: Bit '14' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for D. + return ((code | (1 << 14)) | (1 << 22)) | (1 << 21); // Set bit '22', '21' and '14' to 1. + + default: + break; } - else + break; + + case EA_16BYTE: + switch (fmt) { - assert(insOptsScalable32bitExtends(opt)); - switch (opt) - { - case INS_OPTS_SCALABLE_S_UXTW: - case INS_OPTS_SCALABLE_S_SXTW: - fmt = IF_SVE_JK_4A_B; - break; + case IF_SVE_IH_3A_F: + return code | (1 << 20); // Set bit '20' to 1. - case INS_OPTS_SCALABLE_D_UXTW: - case INS_OPTS_SCALABLE_D_SXTW: - fmt = IF_SVE_JK_4A; - break; + case IF_SVE_II_4A_H: + // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for Q. + return code | (1 << 15); // Set bit '15' to 1. - default: - assert(!"Invalid options for scalable"); - break; - } + default: + break; } break; - case INS_sve_st1h: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isScalableVectorSize(size)); + default: + assert(!"Invalid size for encoding dtype."); + break; + } - if (insOptsScalableStandard(opt)) + assert(!"Invalid instruction format"); + return code; +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * for the 'dtypeh' and 'dtypel' fields. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtypeh_dtypel(instruction ins, + insFormat fmt, + emitAttr size, + code_t code) +{ + switch (fmt) + { + case IF_SVE_IC_3A_A: + switch (size) { - if (sopt == INS_SCALABLE_OPTS_LSL_N) - { - if (isGeneralRegister(reg4)) + case EA_4BYTE: + switch (ins) { - // st1h is reserved for scalable B - assert((ins == INS_sve_st1h) ? insOptsScalableAtLeastHalf(opt) : true); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - fmt = IF_SVE_JD_4A; + case INS_sve_ld1rsh: + return code | (1 << 13); // set bit '13' + + case INS_sve_ld1rw: + return code | (1 << 14); // set bit '14' + + default: + break; } - else + break; + + case EA_8BYTE: + switch (ins) { - assert(isVectorRegister(reg4)); - fmt = IF_SVE_JJ_4B; - } - } - else - { - assert(isVectorRegister(reg4)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_JJ_4B_E; - } - } - else - { - assert(insOptsScalable32bitExtends(opt)); - switch (opt) - { - case INS_OPTS_SCALABLE_S_UXTW: - case INS_OPTS_SCALABLE_S_SXTW: - if (insScalableOptsNone(sopt)) - { - fmt = IF_SVE_JJ_4A_D; - } - else - { - assert(sopt == INS_SCALABLE_OPTS_MOD_N); - fmt = IF_SVE_JJ_4A; - } - break; + case INS_sve_ld1rsh: + return code; - case INS_OPTS_SCALABLE_D_UXTW: - case INS_OPTS_SCALABLE_D_SXTW: - if (insScalableOptsNone(sopt)) - { - fmt = IF_SVE_JJ_4A_C; - } - else - { - assert(sopt == INS_SCALABLE_OPTS_MOD_N); - fmt = IF_SVE_JJ_4A_B; - } - break; + case INS_sve_ld1rw: + return code | (1 << 14) | (1 << 13); // set bits '14' and '13' - default: - assert(!"Invalid options for scalable"); - break; - } + default: + break; + } + break; + + default: + break; } break; - case INS_sve_st1w: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isScalableVectorSize(size)); - - if (insOptsScalableStandard(opt)) + case IF_SVE_IC_3A_B: + switch (size) { - if (sopt == INS_SCALABLE_OPTS_LSL_N) - { - if (isGeneralRegister(reg4)) + case EA_2BYTE: + switch (ins) { - fmt = IF_SVE_JD_4B; + case INS_sve_ld1rh: + return code | (1 << 13); // set bit '13' + + case INS_sve_ld1rsb: + return code | (1 << 24) | (1 << 14); // set bit '24' and '14' + + default: + break; } - else + break; + + case EA_4BYTE: + switch (ins) { - assert(isVectorRegister(reg4)); - fmt = IF_SVE_JJ_4B; + case INS_sve_ld1rh: + return code | (1 << 14); // set bit '14' + + case INS_sve_ld1rsb: + return code | (1 << 24) | (1 << 13); // set bit '24' and '13' + + default: + break; } - } - else - { - assert(isVectorRegister(reg4)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_JJ_4B_E; - } + break; + + case EA_8BYTE: + switch (ins) + { + case INS_sve_ld1rh: + return code | (1 << 14) | (1 << 13); // set bits '14' and '13' + + case INS_sve_ld1rsb: + return code | (1 << 24); // set bit '24' + + default: + break; + } + break; + + default: + break; } - else if (opt == INS_OPTS_SCALABLE_Q) + break; + + case IF_SVE_IC_3A_C: + assert(ins == INS_sve_ld1rb); + switch (size) + { + case EA_1BYTE: + return code; + + case EA_2BYTE: + return code | (1 << 13); // set bit '13' + + case EA_4BYTE: + return code | (1 << 14); // set bit '14' + + case EA_8BYTE: + return code | (1 << 14) | (1 << 13); // set bits '14' and '13' + + default: + break; + } + break; + + default: + break; + } + + assert(!"Unexpected instruction format"); + return code; +} + +/***************************************************************************** + * + * Returns the encoding to select the 4/8-byte width specifier + * at bit location 22 for an Arm64 Sve instruction. + */ +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_R_22(emitAttr size) +{ + if (size == EA_8BYTE) + { + return 0x400000; // set the bit at location 22 + } + + assert(size == EA_4BYTE); + return 0; +} + +/***************************************************************************** + * + * Returns the immediate value for SVE instructions that encode it as a difference + * from tszh:tszl:imm3. + */ +/*static*/ ssize_t emitter::insSveGetImmDiff(const ssize_t imm, const insOpts opt) +{ + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimmFrom1<3>(imm)); + return (8 - imm); + + case INS_OPTS_SCALABLE_H: + assert(isValidUimmFrom1<4>(imm)); + return (16 - imm); + + case INS_OPTS_SCALABLE_S: + assert(isValidUimmFrom1<5>(imm)); + return (32 - imm); + + case INS_OPTS_SCALABLE_D: + assert(isValidUimmFrom1<6>(imm)); + return (64 - imm); + + default: + unreached(); + break; + } + + return 0; +} + +/***************************************************************************** + * + * Returns the two 5-bit signed immediates encoded in the following format: + * njjj jjmi iiii + * - iiiii: the absolute value of imm1 + * - m: 1 if imm1 is negative, 0 otherwise + * - jjjjj: the absolute value of imm2 + * - n: 1 if imm2 is negative, 0 otherwise + */ +/*static*/ ssize_t emitter::insSveEncodeTwoSimm5(ssize_t imm1, ssize_t imm2) +{ + assert(isValidSimm<5>(imm1)); + assert(isValidSimm<5>(imm2)); + ssize_t immOut = 0; + + if (imm1 < 0) + { + // Set bit location 5 to indicate imm1 is negative + immOut |= 0x20; + imm1 *= -1; + } + + if (imm2 < 0) + { + // Set bit location 11 to indicate imm2 is negative + immOut |= 0x800; + imm2 *= -1; + } + + immOut |= imm1; + immOut |= (imm2 << 6); + return immOut; +} + +/***************************************************************************** + * + * Decodes imm into two 5-bit signed immediates, + * using the encoding format from insSveEncodeTwoSimm5. + */ +/*static*/ void emitter::insSveDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2) +{ + assert(imm1 != nullptr); + assert(imm2 != nullptr); + + *imm1 = (imm & 0x1F); + + if ((imm & 0x20) != 0) + { + *imm1 *= -1; + } + + imm >>= 6; + *imm2 = (imm & 0x1F); + + if ((imm & 0x20) != 0) + { + *imm2 *= -1; + } + + assert(isValidSimm<5>(*imm1)); + assert(isValidSimm<5>(*imm2)); +} + +/************************************************************************ + * + * Convert a small immediate float value to an encoded version that matches one-to-one with the instructions. + * The instruction determines the value. + */ + +/*static*/ ssize_t emitter::emitEncodeSmallFloatImm(double immDbl, instruction ins) +{ +#ifdef DEBUG + switch (ins) + { + case INS_sve_fadd: + case INS_sve_fsub: + case INS_sve_fsubr: + assert((immDbl == 0.5) || (immDbl == 1.0)); + break; + + case INS_sve_fmax: + case INS_sve_fmaxnm: + case INS_sve_fmin: + case INS_sve_fminnm: + assert((immDbl == 0) || (immDbl == 1.0)); + break; + + case INS_sve_fmul: + assert((immDbl == 0.5) || (immDbl == 2.0)); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + if (immDbl < 1.0) + { + return 0; + } + return 1; +} + +/************************************************************************ + * + * Convert an encoded small float immediate value. The instruction determines the value. + */ + +/*static*/ double emitter::emitDecodeSmallFloatImm(ssize_t imm, instruction ins) +{ + assert(emitIsValidEncodedSmallFloatImm(imm)); + switch (ins) + { + case INS_sve_fadd: + case INS_sve_fsub: + case INS_sve_fsubr: + if (imm == 0) { - assert(isGeneralRegister(reg4)); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - fmt = IF_SVE_JD_4C; + return 0.5; } else { - assert(insOptsScalable32bitExtends(opt)); - assert(isVectorRegister(reg4)); - switch (opt) - { - case INS_OPTS_SCALABLE_S_UXTW: - case INS_OPTS_SCALABLE_S_SXTW: - if (insScalableOptsNone(sopt)) - { - fmt = IF_SVE_JJ_4A_D; - } - else - { - assert(sopt == INS_SCALABLE_OPTS_MOD_N); - fmt = IF_SVE_JJ_4A; - } - break; - - case INS_OPTS_SCALABLE_D_UXTW: - case INS_OPTS_SCALABLE_D_SXTW: - if (insScalableOptsNone(sopt)) - { - fmt = IF_SVE_JJ_4A_C; - } - else - { - assert(sopt == INS_SCALABLE_OPTS_MOD_N); - fmt = IF_SVE_JJ_4A_B; - } - break; + return 1.0; + } - default: - assert(!"Invalid options for scalable"); - break; - } + case INS_sve_fmax: + case INS_sve_fmaxnm: + case INS_sve_fmin: + case INS_sve_fminnm: + if (imm == 0) + { + return 0.0; + } + else + { + return 1.0; } break; - case INS_sve_st1d: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isScalableVectorSize(size)); - - if (isGeneralRegister(reg4)) + case INS_sve_fmul: + if (imm == 0) { - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - if (opt == INS_OPTS_SCALABLE_Q) - { - fmt = IF_SVE_JD_4C_A; - } - else - { - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_JD_4C; - } + return 0.5; } else { - assert(isVectorRegister(reg4)); + return 2.0; + } + break; - if (opt == INS_OPTS_SCALABLE_D) - { - if (sopt == INS_SCALABLE_OPTS_LSL_N) - { - fmt = IF_SVE_JJ_4B; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_JJ_4B_C; - } - } - else - { - assert(insOptsScalable32bitExtends(opt)); - switch (opt) - { - case INS_OPTS_SCALABLE_D_UXTW: - case INS_OPTS_SCALABLE_D_SXTW: - if (sopt == INS_SCALABLE_OPTS_MOD_N) - { - fmt = IF_SVE_JJ_4A; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_JJ_4A_B; - } - break; + default: + break; + } - default: - assert(!"Invalid options for scalable"); - break; - } - } - } + assert(!"Invalid instruction"); + return 0.0; +} + +/************************************************************************ + * + * Check if the immediate value is a valid encoded small float. + */ + +/*static*/ bool emitter::emitIsValidEncodedSmallFloatImm(size_t imm) +{ + return (imm == 0) || (imm == 1); +} + +/************************************************************************ + * + * Convert a rotation value that is 90 or 270 into a smaller encoding that matches one-to-one with the 'rot' field. + */ + +/*static*/ ssize_t emitter::emitEncodeRotationImm90_or_270(ssize_t imm) +{ + switch (imm) + { + case 90: + return 0; + + case 270: + return 1; + + default: break; + } - case INS_sve_ld1b: - case INS_sve_ld1sb: - case INS_sve_ldff1b: - case INS_sve_ldff1sb: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isScalableVectorSize(size)); - assert(insScalableOptsNone(sopt)); + assert(!"Invalid rotation value"); + return 0; +} - if (isGeneralRegisterOrZR(reg4)) - { - switch (ins) - { - case INS_sve_ldff1b: - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_IG_4A_E; - break; +/************************************************************************ + * + * Convert an encoded rotation value to 90 or 270. + */ - case INS_sve_ldff1sb: - assert(insOptsScalableAtLeastHalf(opt)); - fmt = IF_SVE_IG_4A_D; - break; +/*static*/ ssize_t emitter::emitDecodeRotationImm90_or_270(ssize_t imm) +{ + assert(emitIsValidEncodedRotationImm0_to_270(imm)); + switch (imm) + { + case 0: + return 90; - case INS_sve_ld1sb: - assert(insOptsScalableAtLeastHalf(opt)); - fmt = IF_SVE_IK_4A_F; - break; + case 1: + return 270; - case INS_sve_ld1b: - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_IK_4A_H; - break; + default: + break; + } - default: - assert(!"Invalid instruction"); - break; - } - } - else - { - assert(isVectorRegister(reg4)); + return 0; +} - if (insOptsScalableDoubleWord32bitExtends(opt)) - { - fmt = IF_SVE_HW_4A; - } - else if (insOptsScalableSingleWord32bitExtends(opt)) - { - fmt = IF_SVE_HW_4A_A; - } - else - { - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_HW_4B; - } - } +/************************************************************************ + * + * Check if the immediate value is a valid encoded rotation value for 90 or 270. + */ + +/*static*/ bool emitter::emitIsValidEncodedRotationImm90_or_270(ssize_t imm) +{ + return (imm == 0) || (imm == 1); +} + +/************************************************************************ + * + * Convert a rotation value that is 0, 90, 180 or 270 into a smaller encoding that matches one-to-one with the 'rot' + * field. + */ + +/*static*/ ssize_t emitter::emitEncodeRotationImm0_to_270(ssize_t imm) +{ + switch (imm) + { + case 0: + return 0; + + case 90: + return 1; + + case 180: + return 2; + + case 270: + return 3; + + default: + break; + } + + assert(!"Invalid rotation value"); + return 0; +} + +/************************************************************************ + * + * Convert an encoded rotation value to 0, 90, 180 or 270. + */ + +/*static*/ ssize_t emitter::emitDecodeRotationImm0_to_270(ssize_t imm) +{ + assert(emitIsValidEncodedRotationImm0_to_270(imm)); + switch (imm) + { + case 0: + return 0; + + case 1: + return 90; + + case 2: + return 180; + + case 3: + return 270; + + default: + break; + } + + return 0; +} + +/************************************************************************ + * + * Check if the immediate value is a valid encoded rotation value for 0, 90, 180 or 270. + */ + +/*static*/ bool emitter::emitIsValidEncodedRotationImm0_to_270(ssize_t imm) +{ + return (imm >= 0) && (imm <= 3); +} + +/***************************************************************************** + * + * Returns the encoding to select an insSvePattern + */ +/*static*/ emitter::code_t emitter::insEncodeSvePattern(insSvePattern pattern) +{ + return (code_t)((unsigned)pattern << 5); +} + +/***************************************************************************** + * + * Returns the encoding for an immediate in the SVE variant of dup (indexed) + */ +/*static*/ emitter::code_t emitter::insEncodeSveBroadcastIndex(emitAttr elemsize, ssize_t index) +{ + unsigned lane_bytes = genLog2(elemsize) + 1; + code_t tsz = (1 << (lane_bytes - 1)); + code_t imm = (code_t)index << lane_bytes | tsz; + return insEncodeSplitUimm<23, 22, 20, 16>(imm); +} + +/***************************************************************************** + * + * Append the machine code corresponding to the given SVE instruction descriptor. + */ +BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) +{ + code_t code = 0; + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); + emitAttr size = id->idOpSize(); + + ssize_t imm; + + switch (fmt) + { + // Scalable. + case IF_SVE_AA_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise logical operations (predicated) + case IF_SVE_AB_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) + case IF_SVE_AC_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer divide vectors (predicated) + case IF_SVE_AD_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer min/max/difference (predicated) + case IF_SVE_AE_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer multiply vectors (predicated) + case IF_SVE_AF_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (predicated) + case IF_SVE_AG_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (quadwords) + case IF_SVE_AI_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (predicated) + case IF_SVE_AJ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (quadwords) + case IF_SVE_AK_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer min/max reduction (predicated) + case IF_SVE_AL_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer min/max reduction (quadwords) + case IF_SVE_AN_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise shift by vector (predicated) + case IF_SVE_AO_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise shift by wide elements (predicated) + case IF_SVE_AP_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise unary operations (predicated) + case IF_SVE_AQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer unary operations (predicated) + case IF_SVE_CL_3A: // ........xx...... ...gggnnnnnddddd -- SVE compress active elements + case IF_SVE_CM_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally broadcast element to vector + case IF_SVE_CN_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally extract element to SIMD&FP scalar + case IF_SVE_CP_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy SIMD&FP scalar register to vector + // (predicated) + case IF_SVE_CR_3A: // ........xx...... ...gggnnnnnddddd -- SVE extract element to SIMD&FP scalar register + case IF_SVE_CU_3A: // ........xx...... ...gggnnnnnddddd -- SVE reverse within elements + case IF_SVE_EP_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 integer halving add/subtract (predicated) + case IF_SVE_EQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer pairwise add and accumulate long + case IF_SVE_ER_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 integer pairwise arithmetic + case IF_SVE_ES_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer unary operations (predicated) + case IF_SVE_ET_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating add/subtract + case IF_SVE_EU_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating/rounding bitwise shift left + // (predicated) + case IF_SVE_GR_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 floating-point pairwise operations + case IF_SVE_GS_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction (quadwords) + case IF_SVE_HE_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction + case IF_SVE_HJ_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point serial reduction (predicated) + case IF_SVE_HL_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) + case IF_SVE_HQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point round to integral value + case IF_SVE_HR_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point unary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm or nnnnn + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) + case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + // Scalable with Merge or Zero predicate + case IF_SVE_AH_3A: // ........xx.....M ...gggnnnnnddddd -- SVE constructive prefix (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // nnnnn + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // ddddd + code |= insEncodePredQualifier_16(id->idPredicateReg2Merge()); // M + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable with shift immediate + case IF_SVE_AM_2A: // ........xx...... ...gggxxiiiddddd -- SVE bitwise shift by immediate (predicated) + { + bool isRightShift = emitInsIsVectorRightShift(ins); + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= + insEncodeSveShift_23_to_22_9_to_0(optGetSveElemsize(id->idInsOpt()), isRightShift, imm); // xx, xxiii + dst += emitOutput_Instr(dst, code); + } + break; + + // Scalable, 4 regs. Reg4 in mmmmm. + case IF_SVE_AR_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE integer multiply-accumulate writing addend + // (predicated) + case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) + case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable, 4 regs. Reg4 in aaaaa. + case IF_SVE_AS_4A: // ........xx.mmmmm ...gggaaaaaddddd -- SVE integer multiply-add writing multiplicand + // (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeReg_V_9_to_5(id->idReg4()); // aaaaa + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable, 3 regs, no predicates + case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high + // (unpredicated) + case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) + case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient + case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments + case IF_SVE_BZ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) + case IF_SVE_BZ_3A_A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) + case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads + case IF_SVE_EH_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer dot product (unpredicated) + case IF_SVE_EL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply-add long + case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high + case IF_SVE_EN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add interleaved long + case IF_SVE_EO_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add long + case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp + case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) + case IF_SVE_FL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long + case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide + case IF_SVE_FN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply long + case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved + case IF_SVE_FQ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise permute + case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long + case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate + case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long + case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part + case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) + case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable, 3 regs, no predicates. General purpose source registers + case IF_SVE_BA_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE index generation (register start, register + // increment) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // hh + code |= insEncodeUimm<22, 22>(id->idInsOpt() == INS_OPTS_SCALABLE_D ? 1 : 0); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // hh + dst += emitOutput_Instr(dst, code); + break; + + // Immediate and pattern to general purpose. + case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count + case IF_SVE_BM_1A: // ............iiii ......pppppddddd -- SVE inc/dec register by element count + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeSvePattern(id->idSvePattern()); // ppppp + code |= insEncodeUimm4From1_19_to_16(imm); // iiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BO_1A: // ...........Xiiii ......pppppddddd -- SVE saturating inc/dec register by element count + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeSvePattern(id->idSvePattern()); // ppppp + code |= insEncodeUimm4From1_19_to_16(imm); // iiii + code |= insEncodeSveElemsize_sz_20(id->idOpSize()); // X + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BQ_2A: // ...........iiiii ...iiinnnnnddddd -- SVE extract vector (immediate offset, destructive) + case IF_SVE_BQ_2B: // ...........iiiii ...iiimmmmmddddd -- SVE extract vector (immediate offset, destructive) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn/mmmmm + code |= insEncodeUimm<12, 10>(imm & 0b111); // iii + code |= insEncodeUimm<20, 16>(imm >> 3); // iiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BN_1A: // ............iiii ......pppppddddd -- SVE inc/dec vector by element count + case IF_SVE_BP_1A: // ............iiii ......pppppddddd -- SVE saturating inc/dec vector by element count + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSvePattern(id->idSvePattern()); // ppppp + code |= insEncodeUimm4From1_19_to_16(imm); // iiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BS_1A: // ..............ii iiiiiiiiiiiddddd -- SVE bitwise logical with immediate (unpredicated) + case IF_SVE_BT_1A: // ..............ii iiiiiiiiiiiddddd -- SVE broadcast bitmask immediate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= (imm << 5); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BU_2A: // ........xx..gggg ...iiiiiiiiddddd -- SVE copy floating-point immediate (predicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeImm8_12_to_5(imm); // iiiiiiii + code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BV_2A: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) + case IF_SVE_BV_2A_J: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg + code |= insEncodeImm8_12_to_5(imm); // iiiiiiii + code |= (id->idHasShift() ? 0x2000 : 0); // h + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BV_2B: // ........xx..gggg ...........ddddd -- SVE copy integer immediate (predicated) + // In emitIns, we set this format's instruction to MOV, as that is the preferred disassembly. + // However, passing (MOV, IF_SVE_BV_2B) to emitInsCodeSve will assert with "encoding_found", + // as FMOV is the only instruction associated with this encoding format. + // Thus, always pass FMOV here, and use MOV elsewhere for simplicity. + code = emitInsCodeSve(INS_sve_fmov, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BW_2A: // ........ii.xxxxx ......nnnnnddddd -- SVE broadcast indexed element + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveBroadcastIndex(optGetSveElemsize(id->idInsOpt()), imm); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i + dst += emitOutput_Instr(dst, code); break; - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - case INS_sve_ld1w: - case INS_sve_ldff1w: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isScalableVectorSize(size)); + case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii + dst += emitOutput_Instr(dst, code); + break; - if (isGeneralRegisterOrZR(reg4)) - { - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii + dst += emitOutput_Instr(dst, code); + break; - switch (ins) - { - case INS_sve_ldff1h: - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_IG_4A_G; - break; + case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i + dst += emitOutput_Instr(dst, code); + break; - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - assert(insOptsScalableWords(opt)); - fmt = IF_SVE_IG_4A_F; - break; + case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii + dst += emitOutput_Instr(dst, code); + break; - case INS_sve_ld1w: - assert(insOptsScalableWordsOrQuadwords(opt)); - fmt = IF_SVE_II_4A_H; - break; + case IF_SVE_CC_2A: // ........xx...... ......mmmmmddddd -- SVE insert SIMD&FP scalar register + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; - case INS_sve_ld1sh: - assert(insOptsScalableWords(opt)); - fmt = IF_SVE_IK_4A_G; - break; + case IF_SVE_CD_2A: // ........xx...... ......mmmmmddddd -- SVE insert general register + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; - case INS_sve_ld1h: - assert(insOptsScalableAtLeastHalf(opt)); - fmt = IF_SVE_IK_4A_I; - break; + case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeReg_P_19_to_16(id->idReg3()); // MMMM + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; - default: - assert(!"Invalid instruction"); - break; - } + case IF_SVE_CJ_2A: // ........xx...... .......nnnn.dddd -- SVE reverse predicate elements + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CK_2A: // ................ .......NNNN.DDDD -- SVE unpack predicate elements + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GQ_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision odd elements + code = emitInsCodeSve(ins, fmt); + + if (ins == INS_sve_fcvtnt && id->idInsOpt() == INS_OPTS_D_TO_S) + { + code |= (1 << 22 | 1 << 17); } - else + else if (ins == INS_sve_fcvtlt && id->idInsOpt() == INS_OPTS_S_TO_D) { - assert(isVectorRegister(reg4)); - - if (insOptsScalableDoubleWord32bitExtends(opt)) - { - if (sopt == INS_SCALABLE_OPTS_MOD_N) - { - fmt = IF_SVE_HW_4A_A; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HW_4A_B; - } - } - else if (insOptsScalableSingleWord32bitExtends(opt)) - { - if (sopt == INS_SCALABLE_OPTS_MOD_N) - { - fmt = IF_SVE_HW_4A; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HW_4A_C; - } - } - else - { - assert(opt == INS_OPTS_SCALABLE_D); - if (sopt == INS_SCALABLE_OPTS_LSL_N) - { - fmt = IF_SVE_HW_4B; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_HW_4B_D; - } - } + code |= (1 << 22 | 1 << 17); } + + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + dst += emitOutput_Instr(dst, code); break; - case INS_sve_ld1d: - case INS_sve_ld1sw: - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isScalableVectorSize(size)); + // Scalable to general register. + case IF_SVE_CO_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally extract element to general register + case IF_SVE_CS_3A: // ........xx...... ...gggnnnnnddddd -- SVE extract element to general register + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; - if (isGeneralRegisterOrZR(reg4)) - { - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + // Scalable from general register. + case IF_SVE_CQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy general register to vector (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_Rn(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; - if (opt == INS_OPTS_SCALABLE_Q) - { - assert(reg4 != REG_ZR); - assert(ins == INS_sve_ld1d); - fmt = IF_SVE_II_4A_B; - } - else - { - assert(opt == INS_OPTS_SCALABLE_D); + case IF_SVE_CT_3A: // ................ ...gggnnnnnddddd -- SVE reverse doublewords + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; - switch (ins) - { - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - fmt = IF_SVE_IG_4A; - break; + case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) + case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // VVV + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn/mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; - case INS_sve_ld1d: - assert(reg4 != REG_ZR); - fmt = IF_SVE_II_4A; - break; + case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) + { + regNumber reg4 = (ins == INS_sve_mov ? id->idReg1() : id->idReg4()); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_13_to_10(id->idReg2()); // VVVV + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(reg4); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + } - case INS_sve_ld1sw: - assert(reg4 != REG_ZR); - fmt = IF_SVE_IK_4A; - break; + case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors + case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors + case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match + case IF_SVE_HT_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeReg_V_20_to_16(id->idReg4()); // nnnnn + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CY_3A: // ........xx.iiiii ...gggnnnnn.DDDD -- SVE integer compare with signed immediate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSimm<20, 16>(imm); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CY_3B: // ........xx.iiiii ii.gggnnnnn.DDDD -- SVE integer compare with unsigned immediate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeUimm<20, 14>(imm); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) + case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments + case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long + case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate + case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) + case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product + case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product + case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations + case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long + case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long + case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long + case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm + code |= insEncodeReg_V_9_to_5(id->idReg3()); // kkkkk + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AW_2A: // ........xx.xxiii ......mmmmmddddd -- sve_int_rotate_imm + imm = insSveGetImmDiff(emitGetInsSC(id), id->idInsOpt()); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeUimm<20, 16>(imm & 0b11111); // xxiii + code |= insEncodeUimm<22, 22>(imm >> 5); // x + code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate + // increment) + { + ssize_t imm1; + ssize_t imm2; + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm<9, 5>(imm1); // iiiii + code |= insEncodeSimm<20, 16>(imm2); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register + // increment) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm<9, 5>(emitGetInsSC(id)); // iiiii + code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate + // increment) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeSimm<20, 16>(emitGetInsSC(id)); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm<10, 5>(emitGetInsSC(id)); // iiiiii + code |= insEncodeReg_R_20_to_16(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm<10, 5>(emitGetInsSC(id)); // iiiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg3()); // aaaaa + code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EG_3A: // ...........iimmm ......nnnnnddddd -- SVE two-way dot product (indexed) + case IF_SVE_EY_3A: // ...........iimmm ......nnnnnddddd -- SVE integer dot product (indexed) + case IF_SVE_EZ_3A: // ...........iimmm ......nnnnnddddd -- SVE mixed sign dot product (indexed) + case IF_SVE_FD_3B: // ...........iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) + case IF_SVE_FF_3B: // ...........iimmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) + case IF_SVE_FI_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) + case IF_SVE_GU_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GX_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) + case IF_SVE_GY_3B: // ...........iimmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) + case IF_SVE_GY_3B_D: // ...........iimmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) + case IF_SVE_FK_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm + code |= insEncodeUimm<20, 19>(emitGetInsSC(id)); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FD_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) + case IF_SVE_FF_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) + case IF_SVE_FI_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) + case IF_SVE_GU_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GX_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) + case IF_SVE_FK_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm + code |= insEncodeUimm<20, 19>(imm & 0b11); // ii + code |= insEncodeUimm<22, 22>(imm >> 2); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FE_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) + case IF_SVE_FG_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 integer multiply-add long (indexed) + case IF_SVE_FH_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) + case IF_SVE_FJ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) + case IF_SVE_GY_3A: // ...........iimmm ....i.nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) + case IF_SVE_GZ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE floating-point multiply-add long (indexed) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<11, 11>(imm & 1); // i + code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm + code |= insEncodeUimm<20, 19>(imm >> 1); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FE_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) + case IF_SVE_FG_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply-add long (indexed) + case IF_SVE_FH_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) + case IF_SVE_FJ_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<11, 11>(imm & 1); // i + code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm + code |= insEncodeUimm<20, 19>(imm & 0b10); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EY_3B: // ...........immmm ......nnnnnddddd -- SVE integer dot product (indexed) + case IF_SVE_FD_3C: // ...........immmm ......nnnnnddddd -- SVE2 integer multiply (indexed) + case IF_SVE_FF_3C: // ...........immmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) + case IF_SVE_FI_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) + case IF_SVE_GU_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GX_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply (indexed) + case IF_SVE_FK_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm + + // index is encoded at bit location 20; + // left-shift by one bit so we can reuse insEncodeUimm<20, 19> without modifying bit location 19 + code |= insEncodeUimm<20, 19>(emitGetInsSC(id) << 1); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CZ_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations + case IF_SVE_DA_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE propagate break from previous partition + { + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - default: - assert(!"Invalid instruction"); - break; - } - } - } - else if (insOptsScalableDoubleWord32bitExtends(opt)) + regNumber regm; + switch (ins) { - assert(isVectorRegister(reg4)); + case INS_sve_mov: + case INS_sve_movs: + regm = id->idReg3(); + break; - if (sopt == INS_SCALABLE_OPTS_MOD_N) - { - fmt = IF_SVE_IU_4A; - } - else - { - assert(insScalableOptsNone(sopt)); + case INS_sve_not: + case INS_sve_nots: + regm = id->idReg2(); + break; - if (ins == INS_sve_ld1d) - { - fmt = IF_SVE_IU_4A_C; - } - else - { - fmt = IF_SVE_IU_4A_A; - } - } + default: + regm = id->idReg4(); } - else if (sopt == INS_SCALABLE_OPTS_LSL_N) - { - assert(isVectorRegister(reg4)); - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_IU_4B; + + code |= insEncodeReg_P_19_to_16(regm); // MMMM + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_CZ_4A_A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations + case IF_SVE_CZ_4A_L: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // NNNN + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeReg_P_19_to_16(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CZ_4A_K: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + code |= insEncodeReg_P_19_to_16(id->idReg1()); // DDDD + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DB_3A: // ................ ..gggg.NNNNMDDDD -- SVE partition break condition + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + code |= insEncodePredQualifier_4(id->idPredicateReg2Merge()); // M + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DB_3B: // ................ ..gggg.NNNN.DDDD -- SVE partition break condition + case IF_SVE_DC_3A: // ................ ..gggg.NNNN.MMMM -- SVE propagate break to next partition + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DD_2A: // ................ .......gggg.DDDD -- SVE predicate first active + case IF_SVE_DG_2A: // ................ .......gggg.DDDD -- SVE predicate read from FFR (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // gggg + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DE_1A: // ........xx...... ......ppppp.DDDD -- SVE predicate initialize + code = emitInsCodeSve(ins, fmt); + code |= insEncodeSvePattern(id->idSvePattern()); // ppppp + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DF_2A: // ........xx...... .......VVVV.DDDD -- SVE predicate next active + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // VVVV + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DH_1A: // ................ ............DDDD -- SVE predicate read from FFR (unpredicated) + case IF_SVE_DJ_1A: // ................ ............DDDD -- SVE predicate zero + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DI_2A: // ................ ..gggg.NNNN..... -- SVE predicate test + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_13_to_10(id->idReg1()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DK_3A: // ........xx...... ..gggg.NNNNddddd -- SVE predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GA_2A: // ............iiii ......nnnn.ddddd -- SME2 multi-vec shift narrow + imm = emitGetInsSC(id); + assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); + assert(emitInsIsVectorRightShift(id->idIns())); + assert(isValidVectorShiftAmount(imm, EA_4BYTE, /* rightShift */ true)); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeVectorShift(EA_4BYTE, true /* right-shift */, imm); // iiii + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_6_Times_Two(id->idReg2()); // nnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DL_2A: // ........xx...... .....l.NNNNddddd -- SVE predicate count (predicate-as-counter) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeVectorLengthSpecifier(id); // l + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DM_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec register by predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DN_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec vector by predicate count + case IF_SVE_DP_2A: // ........xx...... .......MMMMddddd -- SVE saturating inc/dec vector by predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DO_2A: // ........xx...... .....X.MMMMddddd -- SVE saturating inc/dec register by predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM + code |= insEncodeVLSElemsize(id->idOpSize()); // X + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DQ_0A: // ................ ................ -- SVE FFR initialise + code = emitInsCodeSve(ins, fmt); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DR_1A: // ................ .......NNNN..... -- SVE FFR write from predicate + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_8_to_5(id->idReg1()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DS_2A: // .........x.mmmmm ......nnnnn..... -- SVE conditionally terminate scalars + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_9_to_5(id->idReg1()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm + code |= insEncodeSveElemsize_R_22(id->idOpSize()); // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FZ_2A: // ................ ......nnnn.ddddd -- SME2 multi-vec extract narrow + case IF_SVE_HG_2A: // ................ ......nnnn.ddddd -- SVE2 FP8 downconverts + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_6(id->idReg2()); // nnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GD_2A: // .........x.xx... ......nnnnnddddd -- SVE2 saturating extract narrow + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + // Bit 23 should not be set by below call + assert(insOptsScalableWide(id->idInsOpt())); + code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<20, 16>(emitGetInsSC(id)); // iii + // Bit 23 should not be set by below call + assert(insOptsScalableWide(id->idInsOpt())); + code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GB_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift right narrow + // Bit 23 should not be set by call to insEncodeSveElemsize_tszh_23_tszl_20_to_19, + // nor should we pass INS_OPTS_SCALABLE_D to insGetImmDiff. + assert(insOptsScalableWide(id->idInsOpt())); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<20, 16>(insSveGetImmDiff(emitGetInsSC(id), id->idInsOpt())); // iii + code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FV_2A: // ........xx...... .....rmmmmmddddd -- SVE2 complex integer add + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeUimm<10, 10>(emitGetInsSC(id)); // r + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FY_3A: // .........x.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long with carry + { + // Size encoding: 1 if INS_OPTS_SCALABLE_D, 0 if INS_OPTS_SCALABLE_S + const ssize_t sizeEncoding = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? 1 : 0; + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm<22, 22>(sizeEncoding); // x + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_GK_2A: // ................ ......mmmmmddddd -- SVE2 crypto destructive binary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GL_1A: // ................ ...........ddddd -- SVE2 crypto unary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DT_3A: // ........xx.mmmmm ...X..nnnnn.DDDD -- SVE integer compare scalar count and limit + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= (id->idOpSize() == EA_8BYTE) ? (1 << 12) : 0; // X + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DW_2A: // ........xx...... ......iiNNN.DDDD -- SVE extract mask predicate from predicate-as-counter + case IF_SVE_DW_2B: // ........xx...... .......iNNN.DDDD -- SVE extract mask predicate from predicate-as-counter + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_7_to_5(id->idReg2()); // NNN + code |= insEncodeUimm<9, 8>(emitGetInsSC(id)); // ii (or i) + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DX_3A: // ........xx.mmmmm ......nnnnn.DDD. -- SVE integer compare scalar count and limit (predicate + // pair) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_1(id->idReg1()); // DDD + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DY_3A: // ........xx.mmmmm ..l...nnnnn..DDD -- SVE integer compare scalar count and limit + // (predicate-as-counter) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeVectorLengthSpecifier(id); // l + code |= insEncodeReg_P_2_to_0(id->idReg1()); // DDD + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DZ_1A: // ........xx...... .............DDD -- sve_int_pn_ptrue + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_2_to_0(id->idReg1()); // DDD + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EA_1A: // ........xx...... ...iiiiiiiiddddd -- SVE broadcast floating-point immediate (unpredicated) + case IF_SVE_ED_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer min/max immediate (unpredicated) + case IF_SVE_EE_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer multiply immediate (unpredicated) + { + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeImm8_12_to_5(imm); // iiiiiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_FA_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) + case IF_SVE_FB_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add (indexed) + case IF_SVE_FC_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex saturating multiply-add (indexed) + { + const ssize_t imm = emitGetInsSC(id); + const ssize_t rot = (imm & 0b11); + const ssize_t index = (imm >> 2); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<11, 10>(rot); // rr + code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm + code |= insEncodeUimm<20, 19>(index); // ii + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_EJ_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer dot product + case IF_SVE_EK_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // rr + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FA_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) + case IF_SVE_FB_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add (indexed) + case IF_SVE_FC_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex saturating multiply-add (indexed) + case IF_SVE_GV_3A: // ...........immmm ....rrnnnnnddddd -- SVE floating-point complex multiply-add (indexed) + { + const ssize_t imm = emitGetInsSC(id); + const ssize_t rot = (imm & 0b11); + const ssize_t index = (imm >> 2); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm + code |= insEncodeUimm<11, 10>(rot); // rr + + // index is encoded at bit location 20; + // left-shift by one bit so we can reuse insEncodeUimm<20, 19> without modifying bit location 19 + code |= insEncodeUimm<20, 19>(index << 1); // i + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_EB_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE broadcast integer immediate (unpredicated) + case IF_SVE_EC_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE integer add/subtract immediate (unpredicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + code |= insEncodeImm8_12_to_5(imm); // iiiiiiii + code |= (id->idHasShift() ? 0x2000 : 0); // h + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EB_1B: // ........xx...... ...........ddddd -- SVE broadcast integer immediate (unpredicated) + // ins is MOV for this encoding, as it is the preferred disassembly, so pass FMOV to emitInsCodeSve + code = emitInsCodeSve(INS_sve_fmov, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DU_3A: // ........xx.mmmmm ......nnnnn.DDDD -- SVE pointer conflict compare + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DV_4A: // ........ix.xxxvv ..NNNN.MMMM.DDDD -- SVE broadcast predicate element + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // NNNN + code |= insEncodeReg_P_8_to_5(id->idReg3()); // MMMM + code |= insEncodeReg_R_17_to_16(id->idReg4()); // vv + code |= insEncodeSveElemsize_tszh_tszl_and_imm(id->idInsOpt(), emitGetInsSC(id)); // ix xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HO_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HO_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + switch (id->idInsOpt()) + { + case INS_OPTS_H_TO_S: + code |= (1 << 16); + break; + case INS_OPTS_H_TO_D: + code |= (1 << 22) | (1 << 16); + break; + case INS_OPTS_S_TO_H: + break; + case INS_OPTS_S_TO_D: + code |= (1 << 22) | (3 << 16); + break; + case INS_OPTS_D_TO_H: + code |= (1 << 22); + break; + case INS_OPTS_D_TO_S: + code |= (1 << 22) | (1 << 17); + break; + default: + unreached(); } - else - { - assert(isVectorRegister(reg4)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(insScalableOptsNone(sopt)); + dst += emitOutput_Instr(dst, code); + break; - if (ins == INS_sve_ld1d) - { - fmt = IF_SVE_IU_4B_D; - } - else - { - fmt = IF_SVE_IU_4B_B; - } + case IF_SVE_HO_3C: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HP_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert to integer + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_H: + code |= (1 << 22) | (1 << 17); + break; + case INS_OPTS_H_TO_S: + code |= (1 << 22) | (1 << 18); + break; + case INS_OPTS_H_TO_D: + code |= (1 << 22) | (3 << 17); + break; + case INS_OPTS_SCALABLE_S: + code |= (1 << 23) | (1 << 18); + break; + case INS_OPTS_S_TO_D: + code |= (3 << 22) | (1 << 18); + break; + case INS_OPTS_D_TO_S: + code |= (3 << 22); + break; + case INS_OPTS_SCALABLE_D: + code |= (3 << 22) | (3 << 17); + break; + default: + unreached(); + break; } + + dst += emitOutput_Instr(dst, code); break; - case INS_sve_ldnt1b: - case INS_sve_ldnt1h: - case INS_sve_ldnt1w: - case INS_sve_ldnt1d: - case INS_sve_ldnt1sb: - case INS_sve_ldnt1sh: - case INS_sve_ldnt1sw: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isScalableVectorSize(size)); + case IF_SVE_HS_3A: // ................ ...gggnnnnnddddd -- SVE integer convert to floating-point + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_H: + code |= (1 << 22) | (1 << 17); + break; + case INS_OPTS_S_TO_H: + code |= (1 << 22) | (1 << 18); + break; + case INS_OPTS_SCALABLE_S: + code |= (1 << 23) | (1 << 18); + break; + case INS_OPTS_S_TO_D: + code |= (1 << 23) | (1 << 22); + break; + case INS_OPTS_D_TO_H: + code |= (1 << 22) | (3 << 17); + break; + case INS_OPTS_D_TO_S: + code |= (3 << 22) | (1 << 18); + break; + case INS_OPTS_SCALABLE_D: + code |= (3 << 22) | (3 << 17); + break; + default: + unreached(); + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IJ_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_E: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_G: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) + case IF_SVE_IL_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_B: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_C: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus + // immediate) + case IF_SVE_IO_3A: // ............iiii ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus + // immediate) + case IF_SVE_IQ_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_IS_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (scalar plus immediate) + case IF_SVE_JE_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_JM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus + // immediate) + case IF_SVE_JN_3C: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - if (isGeneralRegister(reg3)) + switch (ins) { - assert(isGeneralRegister(reg4)); - -#ifdef DEBUG - switch (ins) - { - case INS_sve_ldnt1b: - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); - break; + case INS_sve_ld2b: + case INS_sve_ld2h: + case INS_sve_ld2w: + case INS_sve_ld2d: + case INS_sve_ld2q: + case INS_sve_st2b: + case INS_sve_st2h: + case INS_sve_st2w: + case INS_sve_st2d: + case INS_sve_st2q: + code |= insEncodeSimm4_MultipleOf2_19_to_16(imm); // iiii + break; - case INS_sve_ldnt1h: - assert(opt == INS_OPTS_SCALABLE_H); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - break; + case INS_sve_ld3b: + case INS_sve_ld3h: + case INS_sve_ld3w: + case INS_sve_ld3d: + case INS_sve_ld3q: + case INS_sve_st3b: + case INS_sve_st3h: + case INS_sve_st3w: + case INS_sve_st3d: + case INS_sve_st3q: + code |= insEncodeSimm4_MultipleOf3_19_to_16(imm); // iiii + break; - case INS_sve_ldnt1w: - assert(opt == INS_OPTS_SCALABLE_S); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - break; + case INS_sve_ld4b: + case INS_sve_ld4h: + case INS_sve_ld4w: + case INS_sve_ld4d: + case INS_sve_ld4q: + case INS_sve_st4b: + case INS_sve_st4h: + case INS_sve_st4w: + case INS_sve_st4d: + case INS_sve_st4q: + code |= insEncodeSimm4_MultipleOf4_19_to_16(imm); // iiii + break; - case INS_sve_ldnt1d: - assert(opt == INS_OPTS_SCALABLE_D); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - break; + case INS_sve_ld1rqb: + case INS_sve_ld1rqd: + case INS_sve_ld1rqh: + case INS_sve_ld1rqw: + code |= insEncodeSimm4_MultipleOf16_19_to_16(imm); // iiii + break; - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG + case INS_sve_ld1rob: + case INS_sve_ld1rod: + case INS_sve_ld1roh: + case INS_sve_ld1row: + code |= insEncodeSimm4_MultipleOf32_19_to_16(imm); // iiii + break; - fmt = IF_SVE_IN_4A; - } - else if ((ins == INS_sve_ldnt1d) || (ins == INS_sve_ldnt1sw)) - { - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg3)); - assert(isGeneralRegisterOrZR(reg4)); - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_IX_4A; + default: + code |= insEncodeSimm<19, 16>(imm); // iiii + break; } - else - { - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg3)); - assert(isGeneralRegisterOrZR(reg4)); - assert(insScalableOptsNone(sopt)); - if (opt == INS_OPTS_SCALABLE_S) + if (canEncodeSveElemsize_dtype(ins)) + { + if (ins == INS_sve_ld1w) { - fmt = IF_SVE_IF_4A; + code = insEncodeSveElemsize_dtype_ld1w(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); } else { - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_IF_4A_A; + code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); } } - break; - case INS_sve_ld1rob: - case INS_sve_ld1roh: - case INS_sve_ld1row: - case INS_sve_ld1rod: - case INS_sve_ld1rqb: - case INS_sve_ld1rqh: - case INS_sve_ld1rqw: - case INS_sve_ld1rqd: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isGeneralRegister(reg4)); - assert(isScalableVectorSize(size)); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JD_4A: // .........xxmmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + code |= insEncodeSveElemsize_22_to_21(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JD_4B: // ..........xmmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + code |= insEncodeSveElemsize_sz_21(optGetSveElemsize(id->idInsOpt())); // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JJ_4A: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JJ_4A_B: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // // offsets) + case IF_SVE_JJ_4A_C: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JJ_4A_D: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JK_4A: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit unscaled + // offsets) + case IF_SVE_JK_4A_B: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit + // unscaled offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_S_SXTW: + case INS_OPTS_SCALABLE_D_SXTW: + code |= (1 << 14); // h + break; -#ifdef DEBUG - switch (ins) - { - case INS_sve_ld1rob: - case INS_sve_ld1rqb: - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); + default: break; + } - case INS_sve_ld1roh: - case INS_sve_ld1rqh: - assert(opt == INS_OPTS_SCALABLE_H); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JN_3A: // .........xx.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSimm<19, 16>(imm); // iiii + code |= insEncodeSveElemsize_22_to_21(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JN_3B: // ..........x.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSimm<19, 16>(imm); // iiii + code |= insEncodeSveElemsize_sz_21(optGetSveElemsize(id->idInsOpt())); // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HW_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_HW_4A_A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_HW_4A_B: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_HW_4A_C: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_IU_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_IU_4A_A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_IU_4A_C: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_S_SXTW: + case INS_OPTS_SCALABLE_D_SXTW: + code |= (1 << 22); // h break; - case INS_sve_ld1row: - case INS_sve_ld1rqw: - assert(opt == INS_OPTS_SCALABLE_S); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + default: break; + } - case INS_sve_ld1rod: - case INS_sve_ld1rqd: - assert(opt == INS_OPTS_SCALABLE_D); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HW_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_HW_4B_D: // ...........mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IF_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit gather non-temporal load (vector plus + // scalar) + case IF_SVE_IF_4A_A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit gather non-temporal load (vector plus + // scalar) + case IF_SVE_IW_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 128-bit gather load (vector plus scalar) + case IF_SVE_IX_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 64-bit gather non-temporal load (vector plus + // scalar) + case IF_SVE_IY_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 128-bit scatter store (vector plus scalar) + case IF_SVE_IZ_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit scatter non-temporal store (vector plus + // scalar) + case IF_SVE_IZ_4A_A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit scatter non-temporal store (vector plus + // scalar) + case IF_SVE_JA_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 64-bit scatter non-temporal store (vector plus + // scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IG_4A_D: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus + // scalar) + case IF_SVE_IG_4A_E: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus + // scalar) + case IF_SVE_IG_4A_F: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus + // scalar) + case IF_SVE_IG_4A_G: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus + // scalar) + case IF_SVE_II_4A_H: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) + case IF_SVE_IK_4A_F: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + case IF_SVE_IK_4A_G: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + case IF_SVE_IK_4A_H: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + case IF_SVE_IK_4A_I: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + + if (canEncodeSveElemsize_dtype(ins)) + { + if (ins == INS_sve_ld1w) + { + code = insEncodeSveElemsize_dtype_ld1w(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); + } + else + { + code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); + } + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IG_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus scalar) + case IF_SVE_II_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) + case IF_SVE_II_4A_B: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) + case IF_SVE_IK_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + case IF_SVE_IN_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus scalar) + case IF_SVE_IP_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus scalar) + case IF_SVE_IR_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus + // scalar) + case IF_SVE_IT_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load multiple structures (scalar plus scalar) + case IF_SVE_JB_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus + // scalar) + case IF_SVE_JC_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE store multiple structures (scalar plus scalar) + case IF_SVE_JD_4C: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) + case IF_SVE_JD_4C_A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) + case IF_SVE_JF_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus + // scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IU_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_IU_4B_B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_IU_4B_D: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_JJ_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JJ_4B_C: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JJ_4B_E: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JK_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit unscaled + // offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GP_3A: // ........xx.....r ...gggmmmmmddddd -- SVE floating-point complex add (predicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeSveImm90_or_270_rot(imm); // r + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GT_4A: // ........xx.mmmmm .rrgggnnnnnddddd -- SVE floating-point complex multiply-add (predicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + code |= insEncodeSveImm0_to_270_rot(imm); // rr + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HI_3A: // ........xx...... ...gggnnnnn.DDDD -- SVE floating-point compare with zero + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HM_2A: // ........xx...... ...ggg....iddddd -- SVE floating-point arithmetic with immediate + // (predicated) + { + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeSveSmallFloatImm(imm); // i + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + } + break; + + case IF_SVE_HN_2A: // ........xx...iii ......mmmmmddddd -- SVE floating-point trig multiply-add coefficient + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeUimm<18, 16>(imm); // iii + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HP_3A: // .............xx. ...gggnnnnnddddd -- SVE floating-point convert to integer + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSveElemsize_18_to_17(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HU_4B: // ...........mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HV_4A: // ........xx.aaaaa ...gggmmmmmddddd -- SVE floating-point multiply-accumulate writing + // multiplicand + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeReg_V_20_to_16(id->idReg4()); // aaaaa + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_ID_2A: // ..........iiiiii ...iiinnnnn.TTTT -- SVE load predicate register + case IF_SVE_JG_2A: // ..........iiiiii ...iiinnnnn.TTTT -- SVE store predicate register + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // TTTT + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSimm9h9l_21_to_16_and_12_to_10(imm); // iii + // iiiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IE_2A: // ..........iiiiii ...iiinnnnnttttt -- SVE load vector register + case IF_SVE_JH_2A: // ..........iiiiii ...iiinnnnnttttt -- SVE store vector register + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSimm9h9l_21_to_16_and_12_to_10(imm); // iii + // iiiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm<23, 22>(imm); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm3h3l_23_to_22_and_12(imm); // ii + // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm1_23(imm); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled + // offsets) + case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit + // scaled offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= id->idSvePrfop(); // oooo + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_S_SXTW: + case INS_OPTS_SCALABLE_D_SXTW: + code |= (1 << 22); // h break; default: - assert(!"Invalid instruction"); break; } -#endif // DEBUG - fmt = IF_SVE_IP_4A; + dst += emitOutput_Instr(dst, code); break; - case INS_sve_ld1q: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isGeneralRegisterOrZR(reg4)); - assert(isScalableVectorSize(size)); - assert(opt == INS_OPTS_SCALABLE_Q); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_IW_4A; + case IF_SVE_HY_3B: // ...........mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled + // offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= id->idSvePrfop(); // oooo + dst += emitOutput_Instr(dst, code); break; - case INS_sve_ld2q: - case INS_sve_ld3q: - case INS_sve_ld4q: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isGeneralRegister(reg4)); - assert(isScalableVectorSize(size)); - assert(opt == INS_OPTS_SCALABLE_Q); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - fmt = IF_SVE_IR_4A; + case IF_SVE_IB_3A: // ...........mmmmm ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= id->idSvePrfop(); // oooo + dst += emitOutput_Instr(dst, code); break; - case INS_sve_ld2b: - case INS_sve_ld3b: - case INS_sve_ld4b: - case INS_sve_ld2h: - case INS_sve_ld3h: - case INS_sve_ld4h: - case INS_sve_ld2w: - case INS_sve_ld3w: - case INS_sve_ld4w: - case INS_sve_ld2d: - case INS_sve_ld3d: - case INS_sve_ld4d: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isGeneralRegister(reg4)); - assert(isScalableVectorSize(size)); + case IF_SVE_HZ_2A_B: // ...........iiiii ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= id->idSvePrfop(); // oooo -#ifdef DEBUG - switch (ins) + if (id->idInsOpt() == INS_OPTS_SCALABLE_D) { - case INS_sve_ld2b: - case INS_sve_ld3b: - case INS_sve_ld4b: - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); - break; + code |= (1 << 30); // set bit '30' to make it a double-word + } - case INS_sve_ld2h: - case INS_sve_ld3h: - case INS_sve_ld4h: - assert(opt == INS_OPTS_SCALABLE_H); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + switch (ins) + { + case INS_sve_prfh: + code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii break; - case INS_sve_ld2w: - case INS_sve_ld3w: - case INS_sve_ld4w: - assert(opt == INS_OPTS_SCALABLE_S); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + case INS_sve_prfw: + code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii break; - case INS_sve_ld2d: - case INS_sve_ld3d: - case INS_sve_ld4d: - assert(opt == INS_OPTS_SCALABLE_D); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + case INS_sve_prfd: + code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii break; default: - assert(!"Invalid instruction"); - break; + assert(ins == INS_sve_prfb); } -#endif // DEBUG - - fmt = IF_SVE_IT_4A; + dst += emitOutput_Instr(dst, code); break; - case INS_sve_st1q: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isGeneralRegisterOrZR(reg4)); - assert(isScalableVectorSize(size)); - assert(opt == INS_OPTS_SCALABLE_Q); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_IY_4A; + case IF_SVE_HX_3A_B: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeUimm<20, 16>(imm); // iiiii + code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); + dst += emitOutput_Instr(dst, code); break; - case INS_sve_stnt1b: - case INS_sve_stnt1h: - case INS_sve_stnt1w: - case INS_sve_stnt1d: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isScalableVectorSize(size)); + case IF_SVE_HX_3A_E: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) + case IF_SVE_IV_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit gather load (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); - if (isGeneralRegister(reg3)) + switch (ins) { - assert(isGeneralRegister(reg4)); -#ifdef DEBUG - switch (ins) - { - case INS_sve_stnt1b: - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); - break; + case INS_sve_ld1d: + case INS_sve_ldff1d: + code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii + break; - case INS_sve_stnt1h: - assert(opt == INS_OPTS_SCALABLE_H); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - break; + case INS_sve_ld1w: + case INS_sve_ld1sw: + case INS_sve_ldff1w: + case INS_sve_ldff1sw: + code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii + break; - case INS_sve_stnt1w: - assert(opt == INS_OPTS_SCALABLE_S); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - break; + default: + code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JL_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit scatter store (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii + dst += emitOutput_Instr(dst, code); + break; - case INS_sve_stnt1d: - assert(opt == INS_OPTS_SCALABLE_D); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - break; + case IF_SVE_JI_3A_A: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit scatter store (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG - fmt = IF_SVE_JB_4A; - } - else + switch (ins) { - assert(isVectorRegister(reg3)); - assert(isGeneralRegisterOrZR(reg4)); - assert(isScalableVectorSize(size)); - assert(insScalableOptsNone(sopt)); + case INS_sve_st1h: + code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii + break; - if (opt == INS_OPTS_SCALABLE_S) - { - fmt = IF_SVE_IZ_4A; - } - else - { - assert(opt == INS_OPTS_SCALABLE_D); - if (ins == INS_sve_stnt1d) - { - fmt = IF_SVE_JA_4A; - } - else - { - fmt = IF_SVE_IZ_4A_A; - } - } + case INS_sve_st1w: + code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii + break; + + default: + assert(ins == INS_sve_st1b); + code |= insEncodeUimm<20, 16>(imm); // iiiii + break; } + + dst += emitOutput_Instr(dst, code); break; - case INS_sve_st2b: - case INS_sve_st3b: - case INS_sve_st4b: - case INS_sve_st2h: - case INS_sve_st3h: - case INS_sve_st4h: - case INS_sve_st2w: - case INS_sve_st3w: - case INS_sve_st4w: - case INS_sve_st2d: - case INS_sve_st3d: - case INS_sve_st4d: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isGeneralRegister(reg4)); - assert(isScalableVectorSize(size)); + case IF_SVE_IA_2A: // ..........iiiiii ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= id->idSvePrfop(); // oooo + code |= insEncodeSimm<21, 16>(imm); // iiiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IC_3A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn -#ifdef DEBUG switch (ins) { - case INS_sve_st2b: - case INS_sve_st3b: - case INS_sve_st4b: - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); + case INS_sve_ld1rd: + code |= insEncodeUimm6_MultipleOf8_21_to_16(imm); // iiiiii break; - case INS_sve_st2h: - case INS_sve_st3h: - case INS_sve_st4h: - assert(opt == INS_OPTS_SCALABLE_H); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + default: + assert(ins == INS_sve_ld1rsw); + code |= insEncodeUimm6_MultipleOf4_21_to_16(imm); // iiiiii break; + } - case INS_sve_st2w: - case INS_sve_st3w: - case INS_sve_st4w: - assert(opt == INS_OPTS_SCALABLE_S); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IC_3A_A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element + case IF_SVE_IC_3A_B: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element + case IF_SVE_IC_3A_C: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code = insEncodeSveElemsize_dtypeh_dtypel(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); + + switch (ins) + { + case INS_sve_ld1rw: + code |= insEncodeUimm6_MultipleOf4_21_to_16(imm); // iiiiii break; - case INS_sve_st2d: - case INS_sve_st3d: - case INS_sve_st4d: - assert(opt == INS_OPTS_SCALABLE_D); - assert(sopt == INS_SCALABLE_OPTS_LSL_N); + case INS_sve_ld1rh: + case INS_sve_ld1rsh: + code |= insEncodeUimm6_MultipleOf2_21_to_16(imm); // iiiiii break; default: - assert(!"Invalid instruction"); + code |= insEncodeUimm<21, 16>(imm); // iiiiii break; } -#endif // DEBUG - fmt = IF_SVE_JC_4A; - break; - case INS_sve_st2q: - case INS_sve_st3q: - case INS_sve_st4q: - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isGeneralRegister(reg3)); - assert(isGeneralRegister(reg4)); - assert(isScalableVectorSize(size)); - assert(opt == INS_OPTS_SCALABLE_Q); - fmt = IF_SVE_JF_4A; + dst += emitOutput_Instr(dst, code); break; - case INS_sve_bfmla: - case INS_sve_bfmls: - assert(opt == INS_OPTS_SCALABLE_H); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isVectorRegister(reg4)); - fmt = IF_SVE_HU_4B; + case IF_SVE_BI_2A: // ................ ......nnnnnddddd -- SVE constructive prefix (unpredicated) + case IF_SVE_HH_2A: // ................ ......nnnnnddddd -- SVE2 FP8 upconverts + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); break; - case INS_sve_fmad: - case INS_sve_fmsb: - case INS_sve_fnmad: - case INS_sve_fnmsb: - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isVectorRegister(reg4)); - fmt = IF_SVE_HV_4A; + case IF_SVE_CB_2A: // ........xx...... ......nnnnnddddd -- SVE broadcast general register + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); break; - default: - unreached(); + case IF_SVE_BJ_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point exponential accelerator + case IF_SVE_CG_2A: // ........xx...... ......nnnnnddddd -- SVE reverse vector elements + case IF_SVE_CH_2A: // ........xx...... ......nnnnnddddd -- SVE unpack vector elements + case IF_SVE_HF_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point reciprocal estimate (unpredicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); break; - } - assert(fmt != IF_NONE); - // Use aliases. - switch (ins) - { - case INS_sve_cmple: - std::swap(reg3, reg4); - ins = INS_sve_cmpge; - break; - case INS_sve_cmplo: - std::swap(reg3, reg4); - ins = INS_sve_cmphi; + case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) + case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert + case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveElemsizeWithShift_tszh_tszl_imm3(id->idInsOpt(), imm, + emitInsIsVectorRightShift(ins)); // xx xxiii + dst += emitOutput_Instr(dst, code); break; - case INS_sve_cmpls: - std::swap(reg3, reg4); - ins = INS_sve_cmphs; + + case IF_SVE_BX_2A: // ...........ixxxx ......nnnnnddddd -- sve_int_perm_dupq_i + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveElemsizeWithImmediate_i1_tsz(id->idInsOpt(), imm); // ixxxx + dst += emitOutput_Instr(dst, code); break; - case INS_sve_cmplt: - std::swap(reg3, reg4); - ins = INS_sve_cmpgt; + + case IF_SVE_BY_2A: // ............iiii ......mmmmmddddd -- sve_int_perm_extq + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeUimm<19, 16>(imm); // iiii + dst += emitOutput_Instr(dst, code); break; - case INS_sve_facle: - std::swap(reg3, reg4); - ins = INS_sve_facge; + + default: + assert(!"Unexpected format"); break; - case INS_sve_faclt: - std::swap(reg3, reg4); - ins = INS_sve_facgt; + } + + return dst; +} + +/***************************************************************************** + * + * Prints the encoding for the Extend Type encoding + */ + +void emitter::emitDispSveExtendOpts(insOpts opt) +{ + switch (opt) + { + case INS_OPTS_LSL: + printf("lsl"); break; - case INS_sve_fcmle: - std::swap(reg3, reg4); - ins = INS_sve_fcmge; + + case INS_OPTS_UXTW: + case INS_OPTS_SCALABLE_S_UXTW: + case INS_OPTS_SCALABLE_D_UXTW: + printf("uxtw"); break; - case INS_sve_fcmlt: - std::swap(reg3, reg4); - ins = INS_sve_fcmgt; + + case INS_OPTS_SXTW: + case INS_OPTS_SCALABLE_S_SXTW: + case INS_OPTS_SCALABLE_D_SXTW: + printf("sxtw"); break; + default: + assert(!"Bad value"); break; } +} - instrDesc* id = emitNewInstr(attr); +/***************************************************************************** + * + * Prints the encoding for the Extend Type encoding along with the N value + */ - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); +void emitter::emitDispSveExtendOptsModN(insOpts opt, ssize_t imm) +{ + assert(imm >= 0 && imm <= 3); - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idReg4(reg4); + if (imm == 0 && opt != INS_OPTS_LSL) + { + emitDispSveExtendOpts(opt); + } + else if (imm > 0) + { + emitDispSveExtendOpts(opt); + printf(" #%d", (int)imm); + } +} + +/***************************************************************************** + * + * Prints the encoding for the or LSL encoding along with the N value + * This is for formats that have [, .T, ], [, .T, #N], [, , LSL #N], + * [{, , LSL #N}] + */ +void emitter::emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2, insOpts opt, insFormat fmt) +{ + printf("["); + + if (isVectorRegister(reg1)) + { + // If the overall instruction is working on 128-bit + // registers, the size of this register for + // the mod addr is always 64-bit. + // Example: LD1Q {.Q }, /Z, [.D{, }] + if (opt == INS_OPTS_SCALABLE_Q) + { + emitDispSveReg(reg1, INS_OPTS_SCALABLE_D, reg2 != REG_ZR); + } + else + { + emitDispSveReg(reg1, opt, reg2 != REG_ZR); + } + } + else + { + emitDispReg(reg1, EA_8BYTE, reg2 != REG_ZR); + } + + if (isVectorRegister(reg2)) + { + emitDispSveReg(reg2, opt, false); + } + else if (reg2 != REG_ZR) + { + emitDispReg(reg2, EA_8BYTE, false); + } + + if (insOptsScalable32bitExtends(opt)) + { + emitDispComma(); + emitDispSveExtendOptsModN(opt, insSveGetLslOrModN(ins, fmt)); + } + // Omit 'lsl #N' only if the second register is ZR. + else if ((reg2 != REG_ZR) && insSveIsLslN(ins, fmt)) + { + emitDispComma(); + switch (insSveGetLslOrModN(ins, fmt)) + { + case 4: + printf("lsl #4"); + break; + + case 3: + printf("lsl #3"); + break; + + case 2: + printf("lsl #2"); + break; + + case 1: + printf("lsl #1"); + break; + + default: + assert(!"Invalid instruction"); + break; + } + } + printf("]"); +} + +/***************************************************************************** + * + * Prints the encoding for format [.S{, #}] + */ +void emitter::emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt) +{ + printf("["); + emitDispSveReg(reg1, opt, imm != 0); + if (imm != 0) + { + // This does not have to be printed as hex. + // We only do it because the capstone disassembly displays this immediate as hex. + // We could not modify capstone without affecting other cases. + emitDispImm(imm, false, /* alwaysHex */ true); + } + printf("]"); +} + +/***************************************************************************** + * + * Prints the encoding for format [{, #, MUL VL}] + */ +void emitter::emitDispSveImmMulVl(regNumber reg1, ssize_t imm) +{ + printf("["); + emitDispReg(reg1, EA_8BYTE, imm != 0); + if (imm != 0) + { + emitDispImm(imm, true); + printf("mul vl"); + } + printf("]"); +} + +/***************************************************************************** + * + * Prints the encoding for format [.D{, #}] + */ +void emitter::emitDispSveImmIndex(regNumber reg1, insOpts opt, ssize_t imm) +{ + printf("["); + if (isVectorRegister(reg1)) + { + emitDispSveReg(reg1, opt, imm != 0); + } + else + { + emitDispReg(reg1, EA_8BYTE, imm != 0); + } + if (imm != 0) + { + // This does not have to be printed as hex. + // We only do it because the capstone disassembly displays this immediate as hex. + // We could not modify capstone without affecting other cases. + emitDispImm(imm, false, /* alwaysHex */ (imm > 31)); + } + printf("]"); +} + +//------------------------------------------------------------------------ +// emitDispSveReg: Display a scalable vector register name +// +void emitter::emitDispSveReg(regNumber reg, bool addComma) +{ + assert(isVectorRegister(reg)); + printf(emitSveRegName(reg)); + + if (addComma) + emitDispComma(); +} + +//------------------------------------------------------------------------ +// emitDispSveReg: Display a scalable vector register name with an arrangement suffix +// +void emitter::emitDispSveReg(regNumber reg, insOpts opt, bool addComma) +{ + assert(isVectorRegister(reg)); + printf(emitSveRegName(reg)); + + if (opt != INS_OPTS_NONE) + { + assert(insOptsScalable(opt) || insOptsScalable32bitExtends(opt)); + emitDispArrangement(opt); + } - dispIns(id); - appendToCurIG(id); + if (addComma) + emitDispComma(); } -/***************************************************************************** - * - * Add a SVE instruction referencing four registers and a constant. - */ +//------------------------------------------------------------------------ +// emitDispSveRegIndex: Display a scalable vector register with indexed element +// +void emitter::emitDispSveRegIndex(regNumber reg, ssize_t index, bool addComma) +{ + assert(isVectorRegister(reg)); + printf(emitSveRegName(reg)); + emitDispElementIndex(index, addComma); +} -void emitter::emitInsSve_R_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - regNumber reg4, - ssize_t imm, - insOpts opt /* = INS_OPT_NONE*/) +//------------------------------------------------------------------------ +// emitDispSveConsecutiveRegList: Display a SVE consecutive vector register list +// +void emitter::emitDispSveConsecutiveRegList(regNumber firstReg, unsigned listSize, insOpts opt, bool addComma) { - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; + assert(isVectorRegister(firstReg)); - /* Figure out the encoding format of the instruction */ - switch (ins) + regNumber currReg = firstReg; + + assert(listSize > 0); + + printf("{ "); + // We do not want the short-hand for list size of 1 or 2. + if ((listSize <= 2) || (((unsigned)currReg + listSize - 1) > (unsigned)REG_V31)) { - case INS_sve_fcmla: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(isVectorRegister(reg4)); - assert(isScalableVectorSize(size)); - imm = emitEncodeRotationImm0_to_270(imm); - fmt = IF_SVE_GT_4A; - break; + for (unsigned i = 0; i < listSize; i++) + { + const bool notLastRegister = (i != listSize - 1); + emitDispSveReg(currReg, opt, notLastRegister); + currReg = (currReg == REG_V31) ? REG_V0 : REG_NEXT(currReg); + } + } + else + { + // short-hand. example: { z0.s - z2.s } which is the same as { z0.s, z1.s, z2.s } + emitDispSveReg(currReg, opt, false); + printf(" - "); + emitDispSveReg((regNumber)(currReg + listSize - 1), opt, false); + } + printf(" }"); - case INS_sve_psel: - unreached(); // TODO-SVE: Not yet supported. - assert(insOptsScalableStandard(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // NNNN - assert(isPredicateRegister(reg3)); // MMMM - assert(isGeneralRegister(reg4)); // vv - assert((REG_R12 <= reg4) && (reg4 <= REG_R15)); + if (addComma) + { + emitDispComma(); + } +} - switch (opt) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); - break; +//------------------------------------------------------------------------ +// emitSveRegName: Returns a scalable vector register name. +// +// Arguments: +// reg - A SIMD and floating-point register. +// +// Return value: +// A string that represents a scalable vector register name. +// +const char* emitter::emitSveRegName(regNumber reg) const +{ + assert((reg >= REG_V0) && (reg <= REG_V31)); - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); - break; + int index = (int)reg - (int)REG_V0; - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - break; + return zRegNames[index]; +} - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); - break; +//------------------------------------------------------------------------ +// emitPredicateRegName: Returns a predicate register name. +// +// Arguments: +// reg - A predicate register. +// +// Return value: +// A string that represents a predicate register name. +// +const char* emitter::emitPredicateRegName(regNumber reg, PredicateType ptype) +{ + assert((reg >= REG_P0) && (reg <= REG_P15)); - default: - unreached(); - break; - } + const int index = (int)reg - (int)REG_P0; + const bool usePnRegs = (ptype == PREDICATE_N) || (ptype == PREDICATE_N_SIZED); - fmt = IF_SVE_DV_4A; - break; + return usePnRegs ? pnRegNames[index] : pRegNames[index]; +} - default: - unreached(); - break; +//------------------------------------------------------------------------ +// emitDispPredicateReg: Display a predicate register name with with an arrangement suffix +// +void emitter::emitDispPredicateReg(regNumber reg, PredicateType ptype, insOpts opt, bool addComma) +{ + assert(isPredicateRegister(reg)); + printf(emitPredicateRegName(reg, ptype)); + + if (ptype == PREDICATE_MERGE) + { + printf("/m"); + } + else if (ptype == PREDICATE_ZERO) + { + printf("/z"); + } + else if (ptype == PREDICATE_SIZED || ptype == PREDICATE_N_SIZED) + { + emitDispElemsize(optGetSveElemsize(opt)); } - assert(fmt != IF_NONE); - instrDesc* id = emitNewInstrCns(attr, imm); + if (addComma) + emitDispComma(); +} - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); +//------------------------------------------------------------------------ +// emitDispPredicateRegPair: Display a pair of predicate registers +// +void emitter::emitDispPredicateRegPair(regNumber reg, insOpts opt) +{ + printf("{ "); + emitDispPredicateReg(reg, PREDICATE_SIZED, opt, true); + emitDispPredicateReg((regNumber)((unsigned)reg + 1), PREDICATE_SIZED, opt, false); + printf(" }, "); +} - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idReg4(reg4); +//------------------------------------------------------------------------ +// emitDispLowPredicateReg: Display a low predicate register name with with an arrangement suffix +// +void emitter::emitDispLowPredicateReg(regNumber reg, PredicateType ptype, insOpts opt, bool addComma) +{ + assert(isLowPredicateRegister(reg)); + reg = (regNumber)((((unsigned)reg - REG_PREDICATE_FIRST) & 0x7) + REG_PREDICATE_FIRST); + emitDispPredicateReg(reg, ptype, opt, addComma); +} - dispIns(id); - appendToCurIG(id); +//------------------------------------------------------------------------ +// emitDispLowPredicateRegPair: Display a pair of low predicate registers +// +void emitter::emitDispLowPredicateRegPair(regNumber reg, insOpts opt) +{ + assert(isLowPredicateRegister(reg)); + + printf("{ "); + const unsigned baseRegNum = ((unsigned)reg - REG_PREDICATE_FIRST) & 0x7; + const unsigned regNum = (baseRegNum * 2) + REG_PREDICATE_FIRST; + emitDispPredicateReg((regNumber)regNum, PREDICATE_SIZED, opt, true); + emitDispPredicateReg((regNumber)(regNum + 1), PREDICATE_SIZED, opt, false); + printf(" }, "); +} + +//------------------------------------------------------------------------ +// emitDispVectorLengthSpecifier: Display the vector length specifier +// +void emitter::emitDispVectorLengthSpecifier(instrDesc* id) +{ + assert(id != nullptr); + assert(insOptsScalableStandard(id->idInsOpt())); + + if (id->idVectorLength4x()) + { + printf("vlx4"); + } + else + { + printf("vlx2"); + } } /***************************************************************************** * - * Add a SVE instruction referencing a register, a SVE Pattern. + * Display an insSvePattern */ - -void emitter::emitIns_R_PATTERN( - instruction ins, emitAttr attr, regNumber reg1, insOpts opt, insSvePattern pattern /* = SVE_PATTERN_ALL*/) +void emitter::emitDispSvePattern(insSvePattern pattern, bool addComma) { - insFormat fmt = IF_NONE; + printf("%s", svePatternNames[pattern]); - /* Figure out the encoding format of the instruction */ - switch (ins) + if (addComma) { - case INS_sve_ptrue: - case INS_sve_ptrues: - assert(isPredicateRegister(reg1)); - assert(isScalableVectorSize(attr)); - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_DE_1A; + emitDispComma(); + } +} + +/***************************************************************************** + * + * Display an insSvePrfop + */ +void emitter::emitDispSvePrfop(insSvePrfop prfop, bool addComma) +{ + switch (prfop) + { + case SVE_PRFOP_PLDL1KEEP: + printf("pldl1keep"); break; - default: - unreached(); + case SVE_PRFOP_PLDL1STRM: + printf("pldl1strm"); break; - } // end switch (ins) - assert(fmt != IF_NONE); + case SVE_PRFOP_PLDL2KEEP: + printf("pldl2keep"); + break; - instrDesc* id = emitNewInstr(attr); + case SVE_PRFOP_PLDL2STRM: + printf("pldl2strm"); + break; - id->idIns(ins); - id->idInsFmt(fmt); + case SVE_PRFOP_PLDL3KEEP: + printf("pldl3keep"); + break; - id->idReg1(reg1); - id->idInsOpt(opt); - id->idSvePattern(pattern); + case SVE_PRFOP_PLDL3STRM: + printf("pldl3strm"); + break; - dispIns(id); - appendToCurIG(id); -} + case SVE_PRFOP_PSTL1KEEP: + printf("pstl1keep"); + break; -/***************************************************************************** - * - * Add a SVE instruction referencing a register, a SVE Pattern and an immediate. - */ + case SVE_PRFOP_PSTL1STRM: + printf("pstl1strm"); + break; -void emitter::emitIns_R_PATTERN_I(instruction ins, - emitAttr attr, - regNumber reg1, - insSvePattern pattern, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */) -{ - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; + case SVE_PRFOP_PSTL2KEEP: + printf("pstl2keep"); + break; - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_sve_cntb: - case INS_sve_cntd: - case INS_sve_cnth: - case INS_sve_cntw: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg1)); // ddddd - assert(isValidUimmFrom1<4>(imm)); // iiii - assert(size == EA_8BYTE); - fmt = IF_SVE_BL_1A; + case SVE_PRFOP_PSTL2STRM: + printf("pstl2strm"); break; - case INS_sve_incd: - case INS_sve_inch: - case INS_sve_incw: - case INS_sve_decd: - case INS_sve_dech: - case INS_sve_decw: - assert(isValidUimmFrom1<4>(imm)); // iiii + case SVE_PRFOP_PSTL3KEEP: + printf("pstl3keep"); + break; - if (insOptsNone(opt)) - { - assert(isGeneralRegister(reg1)); // ddddd - assert(size == EA_8BYTE); - fmt = IF_SVE_BM_1A; - } - else - { - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - fmt = IF_SVE_BN_1A; - } + case SVE_PRFOP_PSTL3STRM: + printf("pstl3strm"); break; - case INS_sve_incb: - case INS_sve_decb: - assert(isGeneralRegister(reg1)); // ddddd - assert(isValidUimmFrom1<4>(imm)); // iiii - assert(size == EA_8BYTE); - fmt = IF_SVE_BM_1A; + case SVE_PRFOP_CONST6: + printf("#6"); break; - case INS_sve_sqincb: - case INS_sve_uqincb: - case INS_sve_sqdecb: - case INS_sve_uqdecb: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg1)); // ddddd - assert(isValidUimmFrom1<4>(imm)); // iiii - assert(isValidGeneralDatasize(size)); // X - fmt = IF_SVE_BO_1A; + case SVE_PRFOP_CONST7: + printf("#7"); break; - case INS_sve_sqinch: - case INS_sve_uqinch: - case INS_sve_sqdech: - case INS_sve_uqdech: - case INS_sve_sqincw: - case INS_sve_uqincw: - case INS_sve_sqdecw: - case INS_sve_uqdecw: - case INS_sve_sqincd: - case INS_sve_uqincd: - case INS_sve_sqdecd: - case INS_sve_uqdecd: - assert(isValidUimmFrom1<4>(imm)); // iiii + case SVE_PRFOP_CONST14: + printf("#0xE"); + break; - if (insOptsNone(opt)) - { - assert(isGeneralRegister(reg1)); // ddddd - assert(isValidGeneralDatasize(size)); // X - fmt = IF_SVE_BO_1A; - } - else - { - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isScalableVectorSize(size)); - fmt = IF_SVE_BP_1A; - } + case SVE_PRFOP_CONST15: + printf("#0xF"); break; default: - unreached(); + assert(!"Invalid prfop"); break; + } - } // end switch (ins) - assert(fmt != IF_NONE); + if (addComma) + { + emitDispComma(); + } +} + +/***************************************************************************** + * + * Returns the encoding to set the vector length specifier (vl) for an Arm64 SVE instruction + */ + +/*static*/ emitter::code_t emitter::insEncodeVectorLengthSpecifier(instrDesc* id) +{ + assert(id != nullptr); + assert(insOptsScalableStandard(id->idInsOpt())); + + if (id->idVectorLength4x()) + { + switch (id->idInsFmt()) + { + case IF_SVE_DL_2A: + return 0x400; // set the bit at location 10 + case IF_SVE_DY_3A: + return 0x2000; // set the bit at location 13 + default: + assert(!"Unexpected format"); + break; + } + } - instrDesc* id = emitNewInstrCns(attr, imm); + return 0; +} - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - id->idOpSize(size); +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '19' thru '16' position. + */ - id->idReg1(reg1); - id->idSvePattern(pattern); +/*static*/ emitter::code_t emitter::insEncodeReg_P_19_to_16(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 16; +} - dispIns(id); - appendToCurIG(id); +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '3' thru '0' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_3_to_0(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg; } /***************************************************************************** * - * Add a SVE instruction referencing three registers and a SVE 'prfop'. + * Return an encoding for the specified 'P' register used in '8' thru '5' position. */ -void emitter::emitIns_PRFOP_R_R_R(instruction ins, - emitAttr attr, - insSvePrfop prfop, - regNumber reg1, - regNumber reg2, - regNumber reg3, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +/*static*/ emitter::code_t emitter::insEncodeReg_P_8_to_5(regNumber reg) { - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 5; +} - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_sve_prfb: - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isScalableVectorSize(size)); +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '13' thru '10' position. + */ - if (insOptsScalable32bitExtends(opt)) - { - assert(isVectorRegister(reg3)); +/*static*/ emitter::code_t emitter::insEncodeReg_P_13_to_10(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 10; +} - if (insOptsScalableSingleWord32bitExtends(opt)) - { - fmt = IF_SVE_HY_3A; - } - else - { - assert(insOptsScalableDoubleWord32bitExtends(opt)); - fmt = IF_SVE_HY_3A_A; - } - } - else if (isVectorRegister(reg3)) - { - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_HY_3B; - } - else - { - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg3)); - fmt = IF_SVE_IB_3A; - } - break; +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '12' thru '10' position. + */ - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - assert(isLowPredicateRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isScalableVectorSize(size)); +/*static*/ emitter::code_t emitter::insEncodeReg_P_12_to_10(regNumber reg) +{ + assert(isLowPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 7)); + return ureg << 10; +} - if (sopt == INS_SCALABLE_OPTS_MOD_N) - { - if (insOptsScalableSingleWord32bitExtends(opt)) - { - fmt = IF_SVE_HY_3A; - } - else - { - assert(insOptsScalableDoubleWord32bitExtends(opt)); - fmt = IF_SVE_HY_3A_A; - } - } - else - { - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - if (isVectorRegister(reg3)) - { - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_HY_3B; - } - else - { - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg3)); - fmt = IF_SVE_IB_3A; - } - } - break; +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '7' thru '5' position. + */ - default: - unreached(); - break; +/*static*/ emitter::code_t emitter::insEncodeReg_P_7_to_5(regNumber reg) +{ + assert(isHighPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P8; + assert((ureg >= 0) && (ureg <= 7)); + return ureg << 5; +} - } // end switch (ins) - assert(fmt != IF_NONE); +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '3' thru '1' position. + */ - instrDesc* id = emitNewInstr(attr); +/*static*/ emitter::code_t emitter::insEncodeReg_P_3_to_1(regNumber reg) +{ + assert(isLowPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 1; +} - id->idIns(ins); - id->idInsOpt(opt); - id->idInsFmt(fmt); +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '2' thru '0' position. + */ - id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idSvePrfop(prfop); +/*static*/ emitter::code_t emitter::insEncodeReg_P_2_to_0(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 8) && (ureg <= 15)); + return (ureg - 8) << 0; +} - dispIns(id); - appendToCurIG(id); +/***************************************************************************** + * + * Return an encoding for the specified predicate type used in '16' position. + */ + +/*static*/ emitter::code_t emitter::insEncodePredQualifier_16(bool merge) +{ + return merge ? 1 << 16 : 0; } /***************************************************************************** * - * Add a SVE instruction referencing two registers, a SVE 'prfop' and an immediate. + * Return an encoding for the specified predicate type used in '4' position. */ -void emitter::emitIns_PRFOP_R_R_I(instruction ins, - emitAttr attr, - insSvePrfop prfop, - regNumber reg1, - regNumber reg2, - int imm, - insOpts opt /* = INS_OPTS_NONE */) +/*static*/ emitter::code_t emitter::insEncodePredQualifier_4(bool merge) { - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; + return merge ? 1 << 4 : 0; +} - /* Figure out the encoding format of the instruction */ - switch (ins) +// For the given 'elemsize' returns the 'arrangement' when used in a SVE vector register arrangement. +// Asserts and returns INS_OPTS_NONE if an invalid 'elemsize' is passed +// +/*static*/ insOpts emitter::optGetSveInsOpt(emitAttr elemsize) +{ + switch (elemsize) { - case INS_sve_prfb: - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - assert(isLowPredicateRegister(reg1)); - assert(isScalableVectorSize(size)); + case EA_1BYTE: + return INS_OPTS_SCALABLE_B; - if (isVectorRegister(reg2)) - { - assert(insOptsScalableWords(opt)); + case EA_2BYTE: + return INS_OPTS_SCALABLE_H; -#ifdef DEBUG - switch (ins) - { - case INS_sve_prfb: - assert(isValidUimm<5>(imm)); - break; + case EA_4BYTE: + return INS_OPTS_SCALABLE_S; - case INS_sve_prfh: - assert((isValidUimm_MultipleOf<5, 2>(imm))); - break; + case EA_8BYTE: + return INS_OPTS_SCALABLE_D; - case INS_sve_prfw: - assert((isValidUimm_MultipleOf<5, 4>(imm))); - break; + case EA_16BYTE: + return INS_OPTS_SCALABLE_Q; - case INS_sve_prfd: - assert((isValidUimm_MultipleOf<5, 8>(imm))); - break; + default: + assert(!"Invalid emitAttr for sve vector register"); + return INS_OPTS_NONE; + } +} - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG - fmt = IF_SVE_HZ_2A_B; - } - else - { - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg2)); - assert(isValidSimm<6>(imm)); - fmt = IF_SVE_IA_2A; - } - break; +// For the given 'arrangement' returns the 'elemsize' specified by the SVE vector register arrangement +// asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed +// +/*static*/ emitAttr emitter::optGetSveElemsize(insOpts arrangement) +{ + switch (arrangement) + { + case INS_OPTS_SCALABLE_B: + return EA_1BYTE; + + case INS_OPTS_SCALABLE_H: + return EA_2BYTE; + + case INS_OPTS_SCALABLE_S: + case INS_OPTS_SCALABLE_S_UXTW: + case INS_OPTS_SCALABLE_S_SXTW: + return EA_4BYTE; + + case INS_OPTS_SCALABLE_D: + case INS_OPTS_SCALABLE_D_UXTW: + case INS_OPTS_SCALABLE_D_SXTW: + return EA_8BYTE; + + case INS_OPTS_SCALABLE_Q: + return EA_16BYTE; default: - unreached(); - break; + assert(!"Invalid insOpt for vector register"); + return EA_UNKNOWN; + } +} - } // end switch (ins) - assert(fmt != IF_NONE); +/*static*/ insOpts emitter::optWidenSveElemsizeArrangement(insOpts arrangement) +{ + switch (arrangement) + { + case INS_OPTS_SCALABLE_B: + return INS_OPTS_SCALABLE_H; - instrDesc* id = emitNewInstrCns(attr, imm); + case INS_OPTS_SCALABLE_H: + return INS_OPTS_SCALABLE_S; - id->idIns(ins); - id->idInsOpt(opt); - id->idInsFmt(fmt); + case INS_OPTS_SCALABLE_S: + return INS_OPTS_SCALABLE_D; - id->idReg1(reg1); - id->idReg2(reg2); - id->idSvePrfop(prfop); + default: + assert(!" invalid 'arrangement' value"); + return INS_OPTS_NONE; + } +} - dispIns(id); - appendToCurIG(id); +/*static*/ insOpts emitter::optSveToQuadwordElemsizeArrangement(insOpts arrangement) +{ + switch (arrangement) + { + case INS_OPTS_SCALABLE_B: + return INS_OPTS_16B; + + case INS_OPTS_SCALABLE_H: + return INS_OPTS_8H; + + case INS_OPTS_SCALABLE_S: + return INS_OPTS_4S; + + case INS_OPTS_SCALABLE_D: + return INS_OPTS_2D; + + default: + assert(!" invalid 'arrangement' value"); + return INS_OPTS_NONE; + } +} + +/***************************************************************************** + * + * Expands an option that has different size operands (INS_OPTS_*_TO_*) into + * a pair of scalable options where the first describes the size of the + * destination operand and the second describes the size of the source operand. + */ + +/*static*/ void emitter::optExpandConversionPair(insOpts opt, insOpts& dst, insOpts& src) +{ + dst = INS_OPTS_NONE; + src = INS_OPTS_NONE; + + switch (opt) + { + case INS_OPTS_H_TO_S: + dst = INS_OPTS_SCALABLE_S; + src = INS_OPTS_SCALABLE_H; + break; + case INS_OPTS_S_TO_H: + dst = INS_OPTS_SCALABLE_H; + src = INS_OPTS_SCALABLE_S; + break; + case INS_OPTS_S_TO_D: + dst = INS_OPTS_SCALABLE_D; + src = INS_OPTS_SCALABLE_S; + break; + case INS_OPTS_D_TO_S: + dst = INS_OPTS_SCALABLE_S; + src = INS_OPTS_SCALABLE_D; + break; + case INS_OPTS_H_TO_D: + dst = INS_OPTS_SCALABLE_D; + src = INS_OPTS_SCALABLE_H; + break; + case INS_OPTS_D_TO_H: + dst = INS_OPTS_SCALABLE_H; + src = INS_OPTS_SCALABLE_D; + break; + case INS_OPTS_SCALABLE_H: + dst = INS_OPTS_SCALABLE_H; + src = INS_OPTS_SCALABLE_H; + break; + case INS_OPTS_SCALABLE_S: + dst = INS_OPTS_SCALABLE_S; + src = INS_OPTS_SCALABLE_S; + break; + case INS_OPTS_SCALABLE_D: + dst = INS_OPTS_SCALABLE_D; + src = INS_OPTS_SCALABLE_D; + break; + default: + noway_assert(!"unreachable"); + break; + } + + assert(dst != INS_OPTS_NONE && src != INS_OPTS_NONE); + return; } #endif // TARGET_ARM64