jit: Optimize instructions based on operand types

erts/emulator/Makefile.in

@@ -1140,7 +1140,8 @@ RUN_OBJS += \
 	$(ESOCK_RUN_OBJS)		$(OBJDIR)/erl_flxctr.o \
 	$(OBJDIR)/erl_nfunc_sched.o \
 	$(OBJDIR)/erl_global_literals.o \
-	$(OBJDIR)/beam_file.o
+	$(OBJDIR)/beam_file.o \
+	$(OBJDIR)/beam_types.o
 
 LTTNG_OBJS = $(OBJDIR)/erlang_lttng.o
 

erts/emulator/beam/beam_file.c

@@ -580,6 +580,58 @@ static int parse_line_chunk(BeamFile *beam, IFF_Chunk *chunk) {
     return 1;
 }
 
+/* We assume the presence of a type table to simplify loading, so we'll need to
+ * create a dummy table (with single entry for the "any type") when we don't
+ * have one. */
+static void init_fallback_type_table(BeamFile *beam) {
+    BeamFile_TypeTable *types;
+
+    types = &beam->types;
+    ASSERT(types->entries == NULL);
+
+    types->entries = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
+                                sizeof(types->entries[0]));
+    types->count = 1;
+
+    types->entries[0].type_union = BEAM_TYPE_ANY;
+}
+
+static int parse_type_chunk(BeamFile *beam, IFF_Chunk *chunk) {
+    BeamFile_TypeTable *types;
+    BeamReader reader;
+
+    Sint32 version, count;
+    int i;
+
+    beamreader_init(chunk->data, chunk->size, &reader);
+
+    LoadAssert(beamreader_read_i32(&reader, &version));
+    LoadAssert(version == BEAM_TYPES_VERSION);
+
+    types = &beam->types;
+    ASSERT(types->entries == NULL);
+
+    LoadAssert(beamreader_read_i32(&reader, &count));
+    LoadAssert(CHECK_ITEM_COUNT(count, 0, sizeof(types->entries[0])));
+    LoadAssert(count >= 1);
+
+    types->entries = erts_alloc(ERTS_ALC_T_PREPARED_CODE,
+                                count * sizeof(types->entries[0]));
+    types->count = count;
+
+    for (i = 0; i < count; i++) {
+        const byte *type_data;
+
+        LoadAssert(beamreader_read_bytes(&reader, 2, &type_data));
+        LoadAssert(beam_types_decode(type_data, 2, &types->entries[i]));
+    }
+
+    /* The first entry MUST be the "any type." */
+    LoadAssert(types->entries[0].type_union == BEAM_TYPE_ANY);
+
+    return 1;
+}
+
 static ErlHeapFragment *new_literal_fragment(Uint size)
 {
     ErlHeapFragment *bp;
@@ -807,6 +859,7 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) {
         MakeIffId('L', 'i', 'n', 'e'), /* 9 */
         MakeIffId('L', 'o', 'c', 'T'), /* 10 */
         MakeIffId('A', 't', 'o', 'm'), /* 11 */
+        MakeIffId('T', 'y', 'p', 'e'), /* 12 */
     };
 
     static const int UTF8_ATOM_CHUNK = 0;
@@ -823,6 +876,7 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) {
     static const int LOC_CHUNK = 10;
 #endif
     static const int OBSOLETE_ATOM_CHUNK = 11;
+    static const int TYPE_CHUNK = 12;
 
     static const int NUM_CHUNKS = sizeof(chunk_iffs) / sizeof(chunk_iffs[0]);
 
@@ -911,6 +965,15 @@ beamfile_read(const byte *data, size_t size, BeamFile *beam) {
         }
     }
 
+    if (chunks[TYPE_CHUNK].size > 0) {
+        if (!parse_type_chunk(beam, &chunks[TYPE_CHUNK])) {
+            error = BEAMFILE_READ_CORRUPT_TYPE_TABLE;
+            goto error;
+        }
+    } else {
+        init_fallback_type_table(beam);
+    }
+
     beam->strings.data = chunks[STR_CHUNK].data;
     beam->strings.size = chunks[STR_CHUNK].size;
 
@@ -1040,6 +1103,11 @@ void beamfile_free(BeamFile *beam) {
         beam->lines.names = NULL;
     }
 
+    if (beam->types.entries) {
+        erts_free(ERTS_ALC_T_PREPARED_CODE, beam->types.entries);
+        beam->types.entries = NULL;
+    }
+
     if (beam->static_literals.entries) {
         beamfile_literal_dtor(&beam->static_literals);
     }
@@ -1523,7 +1591,7 @@ static int beamcodereader_read_next(BeamCodeReader *code_reader, BeamOp **out) {
             case 4:
                 /* Literal */
                 {
-                    BeamFile_LiteralTable *literals;
+                    const BeamFile_LiteralTable *literals;
                     TaggedNumber index;
 
                     LoadAssert(beamreader_read_tagged(reader, &index));
@@ -1537,6 +1605,25 @@ static int beamcodereader_read_next(BeamCodeReader *code_reader, BeamOp **out) {
                     raw_arg.tag = TAG_q;
                     raw_arg.word_value = index.word_value;
 
+                    break;
+                }
+            case 5:
+                /* Register with type hint */
+                {
+                    const BeamFile_TypeTable *types;
+                    TaggedNumber index;
+
+                    LoadAssert(beamreader_read_tagged(reader, &raw_arg));
+                    LoadAssert(raw_arg.tag == TAG_x || raw_arg.tag == TAG_y);
+
+                    LoadAssert(beamreader_read_tagged(reader, &index));
+                    LoadAssert(index.tag == TAG_u);
+
+                    types = &(code_reader->file)->types;
+                    LoadAssert(index.word_value < types->count);
+
+                    ERTS_CT_ASSERT(REG_MASK < (1 << 10));
+                    raw_arg.word_value |= index.word_value << 10;
                     break;
                 }
             default:

erts/emulator/beam/beam_file.h

@@ -25,6 +25,7 @@
 
 #include "sys.h"
 #include "atom.h"
+#include "beam_types.h"
 
 #define CHECKSUM_SIZE 16
 
@@ -136,6 +137,13 @@ typedef struct {
     BeamFile_LiteralEntry *entries;
 } BeamFile_LiteralTable;
 
+typedef struct {
+    /* To simplify code that queries types, the first entry (which must be
+     * present) is always the "any type." */
+    Sint32 count;
+    BeamType *entries;
+} BeamFile_TypeTable;
+
 typedef struct {
     IFF_File iff;
 
@@ -151,6 +159,7 @@ typedef struct {
 #endif
     BeamFile_LambdaTable lambdas;
     BeamFile_LineTable lines;
+    BeamFile_TypeTable types;
 
     /* Static literals are those defined in the file, and dynamic literals are
      * those created when loading. The former is positively indexed starting
@@ -190,13 +199,14 @@ enum beamfile_read_result {
     /* Optional chunks */
     BEAMFILE_READ_CORRUPT_LAMBDA_TABLE,
     BEAMFILE_READ_CORRUPT_LINE_TABLE,
-    BEAMFILE_READ_CORRUPT_LITERAL_TABLE
+    BEAMFILE_READ_CORRUPT_LITERAL_TABLE,
+    BEAMFILE_READ_CORRUPT_TYPE_TABLE
 };
 
 typedef struct {
     /* TAG_xyz */
-    int type;
-    BeamInstr val;
+    UWord type;
+    UWord val;
 } BeamOpArg;
 
 typedef struct beamop {

erts/emulator/beam/beam_load.c

@@ -154,6 +154,8 @@ erts_prepare_loading(Binary* magic, Process *c_p, Eterm group_leader,
         BeamLoadError0(stp, "corrupt literal table");
     case BEAMFILE_READ_CORRUPT_LOCALS_TABLE:
         BeamLoadError0(stp, "corrupt locals table");
+    case BEAMFILE_READ_CORRUPT_TYPE_TABLE:
+        BeamLoadError0(stp, "corrupt type table");
     case BEAMFILE_READ_SUCCESS:
         break;
     }

erts/emulator/beam/beam_types.c

@@ -0,0 +1,42 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2021-2021. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "beam_types.h"
+
+#include "global.h"
+#include "erl_message.h"
+#include "external.h"
+
+int beam_types_decode(const byte *data, Uint size, BeamType *out) {
+    int types;
+
+    if (size != 2) {
+        return 0;
+    }
+
+    types = (Uint16)data[0] << 8 | (Uint16)data[1];
+    out->type_union = types;
+
+    return 1;
+}

erts/emulator/beam/beam_types.h

@@ -0,0 +1,97 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2021-2021. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+/**
+ * @description Basic type representation for BEAM instruction operands.
+ * @file beam_types.h
+ * 
+ * While the compiler is good eliminating redundant type tests and simplifying
+ * instructions, we're limited by the available instructions and it's not
+ * always worthwhile to add new variants.
+ * 
+ * The idea behind this module is to allow minor optimizations _inside_
+ * instructions based on what we know about their operand types. For example,
+ * when we know that the source passed to `is_tagged_tuple` is always boxed, we
+ * can skip the boxed check.
+ */
+
+#ifndef _BEAM_TYPES_H
+#define _BEAM_TYPES_H
+
+#include "sys.h"
+
+#define BEAM_TYPES_VERSION 0
+
+#define BEAM_TYPE_NONE               (0)
+
+#define BEAM_TYPE_ATOM               (1 << 0)
+#define BEAM_TYPE_BITSTRING          (1 << 1)
+#define BEAM_TYPE_BS_MATCHSTATE      (1 << 2)
+#define BEAM_TYPE_CONS               (1 << 3)
+#define BEAM_TYPE_FLOAT              (1 << 4)
+#define BEAM_TYPE_FUN                (1 << 5)
+#define BEAM_TYPE_INTEGER            (1 << 6)
+#define BEAM_TYPE_MAP                (1 << 7)
+#define BEAM_TYPE_NIL                (1 << 8)
+#define BEAM_TYPE_PID                (1 << 9)
+#define BEAM_TYPE_PORT               (1 << 10)
+#define BEAM_TYPE_REFERENCE          (1 << 11)
+#define BEAM_TYPE_TUPLE              (1 << 12)
+
+#define BEAM_TYPE_ANY                ((1 << 13) - 1)
+
+#define BEAM_TYPE_MASK_BOXED        \
+    (BEAM_TYPE_BITSTRING |          \
+     BEAM_TYPE_BS_MATCHSTATE |      \
+     BEAM_TYPE_FLOAT |              \
+     BEAM_TYPE_FUN |                \
+     BEAM_TYPE_INTEGER |            \
+     BEAM_TYPE_MAP |                \
+     BEAM_TYPE_PID |                \
+     BEAM_TYPE_PORT |               \
+     BEAM_TYPE_REFERENCE |          \
+     BEAM_TYPE_TUPLE)
+
+#define BEAM_TYPE_MASK_IMMEDIATE    \
+    (BEAM_TYPE_ATOM |               \
+     BEAM_TYPE_INTEGER |            \
+     BEAM_TYPE_NIL |                \
+     BEAM_TYPE_PID |                \
+     BEAM_TYPE_PORT)
+
+#define BEAM_TYPE_MASK_CELL         \
+    (BEAM_TYPE_CONS)
+
+#define BEAM_TYPE_MASK_ALWAYS_IMMEDIATE  \
+    (BEAM_TYPE_MASK_IMMEDIATE & ~(BEAM_TYPE_MASK_BOXED | BEAM_TYPE_MASK_CELL))
+#define BEAM_TYPE_MASK_ALWAYS_BOXED      \
+    (BEAM_TYPE_MASK_BOXED & ~(BEAM_TYPE_MASK_CELL | BEAM_TYPE_MASK_IMMEDIATE))
+#define BEAM_TYPE_MASK_ALWAYS_CELL       \
+    (BEAM_TYPE_MASK_CELL & ~(BEAM_TYPE_MASK_BOXED | BEAM_TYPE_MASK_IMMEDIATE))
+
+typedef struct {
+    /** @brief A set of the possible types (atom, tuple, etc) this term may
+     * be. When a single bit is set, the term will always be of that type. */
+    int type_union;
+} BeamType;
+
+int beam_types_decode(const byte *data, Uint size, BeamType *out);
+
+#endif

erts/emulator/beam/emu/emu_load.c

@@ -831,11 +831,11 @@ int beam_load_emit_op(LoaderState *stp, BeamOp *tmp_op) {
             break;
         case 'x':        /* x(N) */
             BeamLoadVerifyTag(stp, tag_to_letter[tag], *sign);
-            code[ci++] = tmp_op->a[arg].val * sizeof(Eterm);
+            code[ci++] = (tmp_op->a[arg].val & REG_MASK) * sizeof(Eterm);
             break;
         case 'y':        /* y(N) */
             BeamLoadVerifyTag(stp, tag_to_letter[tag], *sign);
-            code[ci++] = (tmp_op->a[arg].val + CP_SIZE) * sizeof(Eterm);
+            code[ci++] = ((tmp_op->a[arg].val & REG_MASK) + CP_SIZE) * sizeof(Eterm);
             break;
         case 'a':                /* Tagged atom */
             BeamLoadVerifyTag(stp, tag_to_letter[tag], *sign);
@@ -865,10 +865,10 @@ int beam_load_emit_op(LoaderState *stp, BeamOp *tmp_op) {
         case 's':        /* Any source (tagged constant or register) */
             switch (tag) {
             case TAG_x:
-                code[ci++] = make_loader_x_reg(tmp_op->a[arg].val);
+                code[ci++] = make_loader_x_reg(tmp_op->a[arg].val & REG_MASK);
                 break;
             case TAG_y:
-                code[ci++] = make_loader_y_reg(tmp_op->a[arg].val + CP_SIZE);
+                code[ci++] = make_loader_y_reg((tmp_op->a[arg].val & REG_MASK) + CP_SIZE);
                 break;
             case TAG_i:
                 code[ci++] = (BeamInstr) make_small((Uint)tmp_op->a[arg].val);
@@ -903,10 +903,10 @@ int beam_load_emit_op(LoaderState *stp, BeamOp *tmp_op) {
         case 'S':   /* Source (x(N), y(N)) */
             switch (tag) {
             case TAG_x:
-                code[ci++] = tmp_op->a[arg].val * sizeof(Eterm);
+                code[ci++] = (tmp_op->a[arg].val & REG_MASK) * sizeof(Eterm);
                 break;
             case TAG_y:
-                code[ci++] = (tmp_op->a[arg].val + CP_SIZE) * sizeof(Eterm) + 1;
+                code[ci++] = ((tmp_op->a[arg].val & REG_MASK) + CP_SIZE) * sizeof(Eterm) + 1;
                 break;
             default:
                 BeamLoadError1(stp, "bad tag %d for destination",