Allow re-initialization and caching of foreign types (#47407)

Co-authored-by: Tim Holy <tim.holy@gmail.com>
Co-authored-by: Max Horn <max@quendi.de>

src/datatype.c

@@ -826,6 +826,22 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name,
     return bt;
 }
 
+JL_DLLEXPORT int jl_reinit_foreign_type(jl_datatype_t *dt,
+                                        jl_markfunc_t markfunc,
+                                        jl_sweepfunc_t sweepfunc)
+{
+    if (!jl_is_foreign_type(dt))
+        return 0;
+    const jl_datatype_layout_t *layout = dt->layout;
+    jl_fielddescdyn_t * desc =
+      (jl_fielddescdyn_t *) ((char *)layout + sizeof(*layout));
+    assert(!desc->markfunc);
+    assert(!desc->sweepfunc);
+    desc->markfunc = markfunc;
+    desc->sweepfunc = sweepfunc;
+    return 1;
+}
+
 JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt)
 {
     return jl_is_datatype(dt) && dt->layout && dt->layout->fielddesc_type == 3;

src/jl_exported_funcs.inc

@@ -343,6 +343,7 @@
     XX(jl_new_code_info_uninit) \
     XX(jl_new_datatype) \
     XX(jl_new_foreign_type) \
+    XX(jl_reinit_foreign_type) \
     XX(jl_new_method_instance_uninit) \
     XX(jl_new_method_table) \
     XX(jl_new_method_uninit) \

src/julia_gcext.h

@@ -49,6 +49,13 @@ JL_DLLEXPORT jl_datatype_t *jl_new_foreign_type(
         int haspointers,
         int large);
 
+
+#define HAVE_JL_REINIT_FOREIGN_TYPE 1
+JL_DLLEXPORT int jl_reinit_foreign_type(
+        jl_datatype_t *dt,
+        jl_markfunc_t markfunc,
+        jl_sweepfunc_t sweepfunc);
+
 JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt);
 
 JL_DLLEXPORT size_t jl_gc_max_internal_obj_size(void);

src/staticdata.c

@@ -77,6 +77,7 @@ External links:
 
 #include "julia.h"
 #include "julia_internal.h"
+#include "julia_gcext.h"
 #include "builtin_proto.h"
 #include "processor.h"
 #include "serialize.h"
@@ -1249,6 +1250,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED
             ios_write(s->s, (char*)v, sizeof(void*) + jl_string_len(v));
             write_uint8(s->s, '\0'); // null-terminated strings for easier C-compatibility
         }
+        else if (jl_is_foreign_type(t) == 1) {
+            jl_error("Cannot serialize instances of foreign datatypes");
+        }
         else if (jl_datatype_nfields(t) == 0) {
             // The object has no fields, so we just snapshot its byte representation
             assert(!t->layout->npointers);
@@ -1438,10 +1442,14 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED
                 if (dt->layout != NULL) {
                     size_t nf = dt->layout->nfields;
                     size_t np = dt->layout->npointers;
-                    size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type);
+                    size_t fieldsize = 0;
+                    uint8_t is_foreign_type = dt->layout->fielddesc_type == 3;
+                    if (!is_foreign_type) {
+                        fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type);
+                    }
                     char *flddesc = (char*)dt->layout;
                     size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize;
-                    if (dt->layout->first_ptr != -1)
+                    if (!is_foreign_type && dt->layout->first_ptr != -1)
                         fldsize += np << dt->layout->fielddesc_type;
                     uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*));
                     write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream
@@ -1450,6 +1458,13 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED
                     arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_datatype_t, layout))); // relocation location
                     arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + layout)); // relocation target
                     ios_write(s->const_data, flddesc, fldsize);
+                    if (is_foreign_type) {
+                        // make sure we have space for the extra hidden pointers
+                        // zero them since they will need to be re-initialized externally
+                        assert(fldsize == sizeof(jl_datatype_layout_t));
+                        jl_fielddescdyn_t dyn = {0, 0};
+                        ios_write(s->const_data, (char*)&dyn, sizeof(jl_fielddescdyn_t));
+                    }
                 }
             }
             else if (jl_is_typename(v)) {

test/gcext/.gitignore

@@ -1,2 +1,3 @@
 /gcext
 /gcext-debug
+/Foreign/deps

test/gcext/DependsOnForeign/Manifest.toml

@@ -0,0 +1,14 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.8.3"
+manifest_format = "2.0"
+project_hash = "e7199d961a5f4ebad68a3deaf5beaa7406a0afcb"
+
+[[deps.Foreign]]
+deps = ["Libdl"]
+path = "../Foreign"
+uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
+version = "0.1.0"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

test/gcext/DependsOnForeign/Project.toml

@@ -0,0 +1,6 @@
+name = "DependsOnForeign"
+uuid = "4b0716e0-dfb5-4e00-8b44-e2685a41517f"
+version = "0.1.0"
+
+[deps]
+Foreign = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"

test/gcext/DependsOnForeign/src/DependsOnForeign.jl

@@ -0,0 +1,14 @@
+module DependsOnForeign
+
+using Foreign
+
+f(obj::FObj) = Base.pointer_from_objref(obj)
+precompile(f, (FObj,))
+
+const FObjRef = Ref{FObj}()
+
+function __init__()
+    FObjRef[] = FObj()
+end
+
+end # module DependsOnForeign

test/gcext/Foreign/Manifest.toml

@@ -0,0 +1,8 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.9.0-DEV"
+manifest_format = "2.0"
+project_hash = "7b70172a2edbdc772ed789e79d4411d7528eae86"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

test/gcext/Foreign/Project.toml

@@ -0,0 +1,6 @@
+name = "Foreign"
+uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
+version = "0.1.0"
+
+[deps]
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

test/gcext/Foreign/deps/foreignlib.c

@@ -0,0 +1,56 @@
+// This file is a part of Julia. License is MIT: https://julialang.org/license
+
+#include "julia.h"
+#include "julia_gcext.h"
+
+// TODO make these atomics
+int nmarks = 0;
+int nsweeps = 0;
+
+uintptr_t mark(jl_ptls_t ptls, jl_value_t *p)
+{
+    nmarks += 1;
+    return 0;
+}
+
+void sweep(jl_value_t *p)
+{
+    nsweeps++;
+}
+
+JL_DLLEXPORT jl_datatype_t *declare_foreign(jl_sym_t* name, jl_module_t *module, jl_datatype_t *parent)
+{
+     return jl_new_foreign_type(name, module, parent, mark, sweep, 1, 0);
+}
+
+// #define GC_MAX_SZCLASS (2032 - sizeof(void *))
+
+JL_DLLEXPORT int reinit_foreign(jl_datatype_t *dt)
+{
+    int ret = jl_reinit_foreign_type(dt, mark, sweep);
+    nmarks = nsweeps = 0;
+    if (ret == 0)
+        return 0;
+    if (dt->layout->npointers != 1)
+        return -1;
+    if (dt->layout->size != 0)
+        return -2;
+    return ret;
+}
+
+JL_DLLEXPORT jl_value_t *allocate_foreign(jl_ptls_t ptls, size_t sz, jl_datatype_t *dt)
+{
+    jl_value_t* obj = jl_gc_alloc_typed(ptls, sz, dt);
+    jl_gc_schedule_foreign_sweepfunc(ptls, obj);
+    return obj;
+}
+
+JL_DLLEXPORT int nmark_counter()
+{
+    return nmarks;
+}
+
+JL_DLLEXPORT int nsweep_counter()
+{
+    return nsweeps;
+}

test/gcext/Foreign/src/Foreign.jl

@@ -0,0 +1,29 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+module Foreign
+
+using Libdl
+
+const foreignlib = joinpath(dirname(joinpath(@__DIR__)), "deps", "foreignlib.$(dlext)")
+
+const FObj = ccall((:declare_foreign, foreignlib), Any, (Any, Any, Any), :FObj, @__MODULE__, Any)
+FObj() = ccall((:allocate_foreign, foreignlib), Any, (Ptr{Cvoid}, Csize_t, Any,), Core.getptls(), sizeof(Ptr{Cvoid}), FObj)::FObj
+
+export FObj
+
+get_nmark()  = ccall((:nmark_counter, foreignlib),  Cint, ())
+get_nsweep() = ccall((:nsweep_counter, foreignlib), Cint, ())
+
+function __init__()
+    @assert ccall((:reinit_foreign, foreignlib), Cint, (Any,), FObj) == 1
+end
+
+allocs(N) = [Foreign.FObj() for _ in 1:N]
+
+function test(N)
+    x = allocs(N)
+    Core.donotdelete(x)
+    x = nothing
+end
+
+end # module Foreign

test/gcext/ForeignObjSerialization/Manifest.toml

@@ -0,0 +1,14 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.8.3"
+manifest_format = "2.0"
+project_hash = "e7199d961a5f4ebad68a3deaf5beaa7406a0afcb"
+
+[[deps.Foreign]]
+deps = ["Libdl"]
+path = "../Foreign"
+uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
+version = "0.1.0"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"

test/gcext/ForeignObjSerialization/Project.toml

@@ -0,0 +1,6 @@
+name = "ForeignObjSerialization"
+uuid = "2c015d96-a6ca-42f0-bc68-f9090de6bc2c"
+version = "0.1.0"
+
+[deps]
+Foreign = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"

test/gcext/ForeignObjSerialization/src/ForeignObjSerialization.jl

@@ -0,0 +1,6 @@
+module ForeignObjSerialization
+
+using Foreign
+const FObjRef = Ref{FObj}(FObj())
+
+end # module ForeignObjSerialization

test/gcext/Makefile

@@ -19,38 +19,54 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
 # get the executable suffix, if any
 EXE := $(suffix $(abspath $(JULIA)))
 
+OS := $(shell uname)
+ifeq ($(OS), Darwin)
+  DYLIB := .dylib
+else
+  DYLIB := .so
+endif
+
 # get compiler and linker flags. (see: `contrib/julia-config.jl`)
 JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "julia-config.jl"))' --
 CPPFLAGS_ADD :=
 CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags)
 LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs)
+DYLIBFLAGS := --shared -fPIC
 
 DEBUGFLAGS += -g
 
 #=============================================================================
 
-release: $(BIN)/gcext$(EXE)
-debug:   $(BIN)/gcext-debug$(EXE)
+release: $(BIN)/gcext$(EXE) $(BIN)/Foreign/deps/foreignlib$(DYLIB)
+debug:   $(BIN)/gcext-debug$(EXE) $(BIN)/Foreign/deps/foreignlib-debug$(DYLIB)
 
 $(BIN)/gcext$(EXE): $(SRCDIR)/gcext.c
 	$(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS)
 
 $(BIN)/gcext-debug$(EXE): $(SRCDIR)/gcext.c
 	$(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS)
 
+$(BIN)/Foreign/deps/foreignlib$(DYLIB): $(SRCDIR)/Foreign/deps/foreignlib.c
+	$(CC) $^ -o $@ $(DYLIBFLAGS) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS)
+
+$(BIN)/Foreign/deps/foreignlib-debug$(DYLIB): $(SRCDIR)/Foreign/deps/foreignlib.c
+	$(CC) $^ -o $@ $(DYLIBFLAGS) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS)
+
 ifneq ($(abspath $(BIN)),$(abspath $(SRCDIR)))
 # for demonstration purposes, our demo code is also installed
 # in $BIN, although this would likely not be typical
 $(BIN)/LocalModule.jl: $(SRCDIR)/LocalModule.jl
 	cp $< $@
 endif
 
-check: $(BIN)/gcext$(EXE) $(BIN)/LocalTest.jl
+check: $(BIN)/gcext$(EXE) $(BIN)/LocalTest.jl $(BIN)/Foreign/deps/foreignlib$(DYLIB)
 	$(JULIA) --depwarn=error $(SRCDIR)/gcext-test.jl $<
 	@echo SUCCESS
 
 clean:
 	-rm -f $(BIN)/gcext-debug$(EXE) $(BIN)/gcext$(EXE)
+	-rm -f $(BIN)/Foreign/deps/foreignlib$(DYLIB)
+	-rm -f $(BIN)/Foreign/deps/foreignlib-debug$(DYLIB)
 
 .PHONY: release debug clean check
 

test/gcext/gcext-test.jl

@@ -2,6 +2,7 @@
 
 # tests the output of the embedding example is correct
 using Test
+using Pkg
 
 if Sys.iswindows()
     # libjulia needs to be in the same directory as the embedding executable or in path
@@ -40,3 +41,35 @@ end
     @test checknum(lines[6], r"([0-9]+) corrupted auxiliary roots",
         n -> n == 0)
 end
+
+@testset "Package with foreign type" begin
+    load_path = copy(LOAD_PATH)
+    push!(LOAD_PATH, joinpath(@__DIR__, "Foreign"))
+    push!(LOAD_PATH, joinpath(@__DIR__, "DependsOnForeign"))
+    try
+        # Force recaching
+        Base.compilecache(Base.identify_package("Foreign"))
+        Base.compilecache(Base.identify_package("DependsOnForeign"))
+
+        push!(LOAD_PATH, joinpath(@__DIR__, "ForeignObjSerialization"))
+        @test_throws ErrorException  Base.compilecache(Base.identify_package("ForeignObjSerialization"), Base.DevNull())
+        pop!(LOAD_PATH)
+
+        (@eval (using Foreign))
+        @test Base.invokelatest(Foreign.get_nmark)  == 0
+        @test Base.invokelatest(Foreign.get_nsweep) == 0
+
+        obj = Base.invokelatest(Foreign.FObj)
+        GC.@preserve obj begin
+            GC.gc(true)
+        end
+        @test Base.invokelatest(Foreign.get_nmark)  > 0
+        @time Base.invokelatest(Foreign.test, 10)
+        GC.gc(true)
+        @test Base.invokelatest(Foreign.get_nsweep) > 0
+        (@eval (using DependsOnForeign))
+        Base.invokelatest(DependsOnForeign.f, obj)
+    finally
+        copy!(LOAD_PATH, load_path)
+    end
+end