From 0d326ed48488bb515a34bf61c3ecd2fdad1e874b Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Sat, 6 Jul 2024 15:23:59 +0100 Subject: [PATCH 1/8] Build and install threads.cmxs Crucially, the corrects the flags used for creating a DLL on Windows, allowing threads.cmxs to be loaded in ocamlnat. --- otherlibs/systhreads/Makefile | 35 ++++++++++++++++++++++----------- otherlibs/systhreads/st_stubs.c | 7 ++----- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/otherlibs/systhreads/Makefile b/otherlibs/systhreads/Makefile index 64d41cc19d66..06430da3b1f8 100644 --- a/otherlibs/systhreads/Makefile +++ b/otherlibs/systhreads/Makefile @@ -39,9 +39,6 @@ LIBNAME=threads # That's why this dependency is handled in the Makefile directly # and removed from the output of the C compiler during make depend -BYTECODE_C_OBJS=st_stubs.b.$(O) -NATIVECODE_C_OBJS=st_stubs.n.$(O) - THREADS_SOURCES = thread.ml event.ml THREADS_BCOBJS = $(THREADS_SOURCES:.ml=.cmo) @@ -55,15 +52,20 @@ all: lib$(LIBNAME).$(A) $(LIBNAME).cma $(CMIFILES) allopt: lib$(LIBNAME)nat.$(A) $(LIBNAME).cmxa $(CMIFILES) +ifeq "$(NATDYNLINK)" "true" +allopt: $(LIBNAME).cmxs +endif + lib$(LIBNAME).$(A): OC_CFLAGS = $(OC_BYTECODE_CFLAGS) -lib$(LIBNAME).$(A): $(BYTECODE_C_OBJS) - $(V_OCAMLMKLIB)$(MKLIB) -o $(LIBNAME) $(BYTECODE_C_OBJS) +lib$(LIBNAME).$(A): st_stubs.b.$(O) st_stubs_shared.b.$(O) + @$(MKLIB) -o $(LIBNAME) st_stubs_shared.b.$(O) + $(V_OCAMLMKLIB)$(MKLIB) -custom -o $(LIBNAME) $< lib$(LIBNAME)nat.$(A): OC_CFLAGS = $(OC_NATIVE_CFLAGS) -lib$(LIBNAME)nat.$(A): $(NATIVECODE_C_OBJS) - $(V_OCAMLMKLIB)$(MKLIB) -o $(LIBNAME)nat $^ +lib$(LIBNAME)nat.$(A): st_stubs.n.$(O) + $(V_OCAMLMKLIB)$(MKLIB) -custom -o $(LIBNAME)nat $^ $(LIBNAME).cma: $(THREADS_BCOBJS) $(V_OCAMLMKLIB)$(MKLIB) -o $(LIBNAME) -ocamlc '$(CAMLC)' -linkall $^ @@ -72,14 +74,20 @@ $(LIBNAME).cma: $(THREADS_BCOBJS) $(LIBNAME).cmxa: $(THREADS_NCOBJS) $(V_LINKOPT)$(CAMLOPT) -linkall -a -cclib -lthreadsnat -o $@ $^ -# The following lines produce two object files st_stubs.b.$(O) and -# st_stubs.n.$(O) from the same source file st_stubs.c (it is compiled -# twice, each time with different options). +st_stubs_shared.n.$(O): OC_CFLAGS = $(OC_NATIVE_CFLAGS) + +$(LIBNAME).cmxs: $(THREADS_NCOBJS) st_stubs_shared.n.$(O) + $(V_LINKOPT)$(CAMLOPT) -linkall -shared -o $@ $^ + +# The following lines produce object files based on st_stubs.c. Four objects are +# produced - a static and shared version in both bytecode and native versions. + +st_stubs_shared.%.$(O): OC_CPPFLAGS += -DSYSTHREADS_SHARED ifeq "$(COMPUTE_DEPS)" "true" -st_stubs.%.$(O): st_stubs.c +st_stubs%.$(O): st_stubs.c else -st_stubs.%.$(O): st_stubs.c $(RUNTIME_HEADERS) $(wildcard *.h) +st_stubs%.$(O): st_stubs.c $(RUNTIME_HEADERS) $(wildcard *.h) endif $(V_CC)$(CC) $(OC_CFLAGS) $(CFLAGS) $(OC_CPPFLAGS) $(CPPFLAGS) \ $(OUTPUTOBJ)$@ -c $< @@ -121,6 +129,9 @@ installopt: $(INSTALL_DATA) \ $(THREADS_NCOBJS) threads.cmxa threads.$(A) \ "$(INSTALL_THREADSLIBDIR)" +ifeq "$(NATDYNLINK)" "true" + $(INSTALL_PROG) $(LIBNAME).cmxs "$(INSTALL_THREADSLIBDIR)" +endif %.cmi: %.mli $(V_OCAMLC)$(CAMLC) -c $(COMPFLAGS) $< diff --git a/otherlibs/systhreads/st_stubs.c b/otherlibs/systhreads/st_stubs.c index 5f2b3abcd525..a91396dcfd1b 100644 --- a/otherlibs/systhreads/st_stubs.c +++ b/otherlibs/systhreads/st_stubs.c @@ -15,16 +15,13 @@ #define CAML_INTERNALS -#if defined(_WIN32) && !defined(NATIVE_CODE) && !defined(_MSC_VER) +#if defined(_WIN32) && defined(SYSTHREADS_SHARED) /* Ensure that pthread.h marks symbols __declspec(dllimport) so that they can be picked up from the runtime (which will have linked winpthreads statically). mingw-w64 11.0.0 introduced WINPTHREADS_USE_DLLIMPORT to do this explicitly; prior versions co-opted this on the internal DLL_EXPORT, but this is ignored in 11.0 and later unless IN_WINPTHREAD is also defined, so we can safely - define both to support both versions. - When compiling with MSVC, we currently link directly the winpthreads objects - into our runtime, so we do not want to mark its symbols with - __declspec(dllimport). */ + define both to support both versions. */ #define WINPTHREADS_USE_DLLIMPORT #define DLL_EXPORT #endif From fe197f13a92ef228c40ca7e9b76e4607d441b591 Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Sun, 14 Jul 2024 12:21:56 +0200 Subject: [PATCH 2/8] Add Config.comprmarsh_c_libraries --- utils/config.fixed.ml | 1 + utils/config.generated.ml.in | 1 + utils/config.mli | 6 ++++++ 3 files changed, 8 insertions(+) diff --git a/utils/config.fixed.ml b/utils/config.fixed.ml index 4082a513aab1..4d7e98c614cb 100644 --- a/utils/config.fixed.ml +++ b/utils/config.fixed.ml @@ -35,6 +35,7 @@ let bytecomp_c_libraries = "" let bytecomp_c_compiler = "" let native_c_compiler = c_compiler let native_c_libraries = "" +let comprmarsh_c_libraries = "" let native_ldflags = "" let native_pack_linker = boot_cannot_call "the linker" let default_rpath = "" diff --git a/utils/config.generated.ml.in b/utils/config.generated.ml.in index ee84889e9336..194479cfaa88 100644 --- a/utils/config.generated.ml.in +++ b/utils/config.generated.ml.in @@ -44,6 +44,7 @@ let bytecomp_c_compiler = let native_c_compiler = c_compiler ^ " " ^ native_cflags ^ " " ^ native_cppflags let native_c_libraries = {@QS@|@cclibs@|@QS@} +let comprmarsh_c_libraries = {@QS@|@zstd_libs@|@QS@} let native_ldflags = {@QS@|@native_ldflags@|@QS@} let native_pack_linker = {@QS@|@PACKLD@|@QS@} let default_rpath = {@QS@|@rpath@|@QS@} diff --git a/utils/config.mli b/utils/config.mli index cea3dc057dea..3227d4e14e09 100644 --- a/utils/config.mli +++ b/utils/config.mli @@ -65,6 +65,12 @@ val bytecomp_c_libraries: string val native_c_libraries: string (** The C libraries to link with native-code programs *) +val comprmarsh_c_libraries: string +(** The C libraries needed with -lcomprmarsh (should appear before + {!native_c_libraries} in a call to the C compiler) + + @since 5.4 *) + val native_ldflags : string (* Flags to pass to the system linker *) From 5271b61fbd76ebdebed23e95f8f27a1e0ebc1994 Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Fri, 5 Jul 2024 09:26:24 +0100 Subject: [PATCH 3/8] Add a test for the compiler "in-prefix" --- .depend | 19 + .gitignore | 2 + Makefile | 16 + configure | 8 + configure.ac | 5 + driver/compmisc.ml | 15 +- driver/compmisc.mli | 2 + testsuite/in_prefix/Makefile.test | 46 ++ testsuite/in_prefix/README.md | 138 ++++ testsuite/tools/main_in_c.c | 23 + testsuite/tools/test_in_prefix.ml | 1079 ++++++++++++++++++++++++++++ testsuite/tools/test_in_prefix.mli | 13 + 12 files changed, 1362 insertions(+), 4 deletions(-) create mode 100644 testsuite/in_prefix/Makefile.test create mode 100644 testsuite/in_prefix/README.md create mode 100644 testsuite/tools/main_in_c.c create mode 100644 testsuite/tools/test_in_prefix.ml create mode 100644 testsuite/tools/test_in_prefix.mli diff --git a/.depend b/.depend index 4583c8e5399a..c997f507c94c 100644 --- a/.depend +++ b/.depend @@ -10578,6 +10578,25 @@ testsuite/tools/parsecmmaux.cmi : \ parsing/location.cmi \ lambda/debuginfo.cmi \ middle_end/backend_var.cmi +testsuite/tools/test_in_prefix.cmo : \ + otherlibs/unix/unix.cmi \ + utils/misc.cmi \ + utils/config.cmi \ + driver/compmisc.cmi \ + file_formats/cmx_format.cmi \ + utils/ccomp.cmi \ + bytecomp/bytesections.cmi \ + testsuite/tools/test_in_prefix.cmi +testsuite/tools/test_in_prefix.cmx : \ + otherlibs/unix/unix.cmx \ + utils/misc.cmx \ + utils/config.cmx \ + driver/compmisc.cmx \ + file_formats/cmx_format.cmi \ + utils/ccomp.cmx \ + bytecomp/bytesections.cmx \ + testsuite/tools/test_in_prefix.cmi +testsuite/tools/test_in_prefix.cmi : otherlibs/dynlink/byte/dynlink.cmo : \ otherlibs/dynlink/dynlink_types.cmi \ otherlibs/dynlink/byte/dynlink_symtable.cmi \ diff --git a/.gitignore b/.gitignore index 5cec38960f9f..2d0262ea2bcc 100644 --- a/.gitignore +++ b/.gitignore @@ -276,6 +276,8 @@ META /testsuite/tools/parsecmm.ml /testsuite/tools/parsecmm.mli /testsuite/tools/parsecmm.output +/testsuite/tools/test_in_prefix +/testsuite/tools/test_in_prefix.opt /tools/ocamldep /tools/ocamldep.opt diff --git a/Makefile b/Makefile index f2a441d4882d..a09ea3669e85 100644 --- a/Makefile +++ b/Makefile @@ -1992,6 +1992,19 @@ $(asmgen_OBJECT): $(asmgen_SOURCE) $(V_ASM)$(ASPP) $(OC_ASPPFLAGS) -o $@ $< || $(ASPP_ERROR) endif +test_in_prefix_SOURCES = \ + $(addprefix testsuite/tools/,test_in_prefix.mli test_in_prefix.ml) +test_in_prefix_LIBRARIES = otherlibs/unix/unix compilerlibs/ocamlcommon + +testsuite/tools/test_in_prefi%: CAMLC = $(BEST_OCAMLC) $(STDLIBFLAGS) + +testsuite/tools/test_in_prefix$(EXE): OC_BYTECODE_LINKFLAGS += -custom + +testsuite/tools/test_in_prefi%: CAMLOPT = $(BEST_OCAMLOPT) $(STDLIBFLAGS) + +testsuite/tools/test_in_prefix.%: \ + OC_COMMON_COMPFLAGS += -rectypes + ocamltest/ocamltest$(EXE): OC_BYTECODE_LINKFLAGS += -custom -g ocamltest/ocamltest$(EXE): ocamlc ocamlyacc ocamllex @@ -2039,6 +2052,9 @@ partialclean:: rm -f $(addprefix testsuite/tools/*.,cm* o obj a lib) rm -f testsuite/tools/codegen testsuite/tools/codegen.exe rm -f testsuite/tools/expect testsuite/tools/expect.exe + rm -f testsuite/tools/test_in_prefix testsuite/tools/test_in_prefix.exe + rm -f testsuite/tools/test_in_prefix.opt \ + testsuite/tools/test_in_prefix.opt.exe rm -f testsuite/tools/lexcmm.ml rm -f $(addprefix testsuite/tools/parsecmm., ml mli output) diff --git a/configure b/configure index c98e1f01784a..69efedff733e 100755 --- a/configure +++ b/configure @@ -20608,6 +20608,14 @@ case $enable_ocamltest,true in #( ocamltest_target=ocamltest ocamltest_opt_target=ocamltest.opt ocamltest='ocamltest' + if $native_compiler +then : + optional_native_tools="$optional_native_tools \ +testsuite/tools/test_in_prefix.opt" +else $as_nop + optional_bytecode_tools="$optional_bytecode_tools \ +testsuite/tools/test_in_prefix" +fi testsuite_tools='testsuite/tools/codegen testsuite/tools/expect' optional_bytecode_tools="$optional_bytecode_tools $testsuite_tools" ;; #( *) : diff --git a/configure.ac b/configure.ac index f180632cc85a..5249b69d4d42 100644 --- a/configure.ac +++ b/configure.ac @@ -2500,6 +2500,11 @@ AS_CASE([$enable_ocamltest,OCAML__DEVELOPMENT_VERSION], ocamltest_target=ocamltest ocamltest_opt_target=ocamltest.opt ocamltest='ocamltest' + AS_IF([$native_compiler], + [optional_native_tools="$optional_native_tools \ +testsuite/tools/test_in_prefix.opt"], + [optional_bytecode_tools="$optional_bytecode_tools \ +testsuite/tools/test_in_prefix"]) testsuite_tools='testsuite/tools/codegen testsuite/tools/expect' optional_bytecode_tools="$optional_bytecode_tools $testsuite_tools"], [build_ocamltest=false diff --git a/driver/compmisc.ml b/driver/compmisc.ml index d4100900380c..b6ee79ebe3ae 100644 --- a/driver/compmisc.ml +++ b/driver/compmisc.ml @@ -28,7 +28,9 @@ let auto_include find_in_dir fn = then the directories specified with the -H option (in command line order). *) -let init_path ?(auto_include=auto_include) ?(dir="") () = +let reinit_path ?(auto_include=auto_include) + ?(standard_library=Config.standard_library) + ?(dir="") () = let visible = if !Clflags.use_threads then "+threads" :: !Clflags.include_dirs else @@ -44,19 +46,24 @@ let init_path ?(auto_include=auto_include) ?(dir="") () = !Compenv.first_include_dirs] in let visible = - List.map (Misc.expand_directory Config.standard_library) visible + List.map (Misc.expand_directory standard_library) visible in let visible = + let std_include = + if !Clflags.no_std_include then [] else [standard_library] + in (if !Clflags.no_cwd then [] else [dir]) - @ List.rev_append visible (Clflags.std_include_dir ()) + @ List.rev_append visible std_include in let hidden = - List.rev_map (Misc.expand_directory Config.standard_library) + List.rev_map (Misc.expand_directory standard_library) !Clflags.hidden_include_dirs in Load_path.init ~auto_include ~visible ~hidden; Env.reset_cache () +let init_path = reinit_path ?standard_library:None + (* Return the initial environment in which compilation proceeds. *) (* Note: do not do init_path() in initial_env, this breaks diff --git a/driver/compmisc.mli b/driver/compmisc.mli index 7b5e1808f0ca..69793899ee76 100644 --- a/driver/compmisc.mli +++ b/driver/compmisc.mli @@ -15,6 +15,8 @@ val init_path : ?auto_include:Load_path.auto_include_callback -> ?dir:string -> unit -> unit +val reinit_path : ?auto_include:Load_path.auto_include_callback + -> ?standard_library:string -> ?dir:string -> unit -> unit val initial_env : unit -> Env.t (* Support for flags that can also be set from an environment variable *) diff --git a/testsuite/in_prefix/Makefile.test b/testsuite/in_prefix/Makefile.test new file mode 100644 index 000000000000..dc0a226a73f4 --- /dev/null +++ b/testsuite/in_prefix/Makefile.test @@ -0,0 +1,46 @@ +#************************************************************************** +#* * +#* OCaml * +#* * +#* David Allsopp, Tarides * +#* * +#* Copyright 2024 David Allsopp Ltd. * +#* * +#* All rights reserved. This file is distributed under the terms of * +#* the GNU Lesser General Public License version 2.1, with the * +#* special exception on linking described in the file LICENSE. * +#* * +#************************************************************************** + +ROOTDIR = ../.. + +include $(ROOTDIR)/Makefile.common + +ifeq "$(NATIVE_COMPILER)" "true" +DRIVER = ../tools/test_in_prefix.opt$(EXE) +else +DRIVER = ../tools/test_in_prefix$(EXE) +endif + +default: $(DRIVER) + @$< --bindir "$(BINDIR)" --libdir "$(LIBDIR)" --summary + @$(error In order to execute the test, run $(MAKE) test-in-prefix) + +test-in-prefix: $(DRIVER) ../tools/main_in_c.$(O) + @$< --bindir "$(BINDIR)" --libdir "$(LIBDIR)" \ + $(call bool_to_with, shared, $(SUPPORTS_SHARED_LIBRARIES)) \ + $(call bool_to_with, ocamlnat, $(INSTALL_OCAMLNAT)) \ + $(call bool_to_with, ocamlopt, $(NATIVE_COMPILER)) \ + $(OTHERLIBRARIES) + +SCRUB_ENV = \ + CAML_LD_LIBRARY_PATH OCAMLLIB CAMLLIB OCAMLPARAM OCAMLRUNPARAM CAMLRUNPARAM + +# Generates --without-$(1) if $(2) = false or --with-$(1) otherwise +bool_to_with = --with$(if $(filter false,$(2)),out)-$(strip $(1)) + +../tools/test_in_prefix$(EXE) \ +../tools/test_in_prefix.opt$(EXE): ../tools/test_in_prefix.ml + +../tools/%: + $(MAKE) -C $(ROOTDIR) testsuite/tools/$* diff --git a/testsuite/in_prefix/README.md b/testsuite/in_prefix/README.md new file mode 100644 index 000000000000..05afab6e7ed0 --- /dev/null +++ b/testsuite/in_prefix/README.md @@ -0,0 +1,138 @@ +# "in-prefix" compiler testing + +This directory performs tests on the compiler after it has been installed to the +prefix given to `configure`. The test requires at least the Unix library to have +been compiled, and only works if bindir and libdir share a common root directory +(e.g. /usr/bin and /usr/lib/ocaml share a common root of /usr, but /bin and +/lib/ocaml do not share a common root directory). The test needs to be able to +rename this common root directory by adding a suffix ".new". For this reasoa, +the test intentionally uses a non-standard Makefile filename and a specific +target giving `make -f Makefile.test -C testsuite/in_prefix test-in-prefix`. The +test should always restore the common root directory to its original name. + +At its most basic level, the test validates the `install` target of the build +system. For this reason, the test driver use command line parameters to be +given the expected configuration of the compiler, rather than probing it (for +example `--with-ocamlopt` is passed, rather than relying on the detection of the +`ocamlopt` binary). + +## Testing + +The test battery consists of six individual tests: + +1. Loading libraries in the bytecode toplevel (skipped for `--disable-shared`) +2. Loading libraries in the native toplevel (skipped for + `--disable-native-toplevel`) +3. Loading libraries with `dynlink.cma` (skipped for `--disable-shared`) +4. Loading libraries with `dynlink.cmxa` (skipped for `--disabled-shared` or + `--disable-native-compiler`) +5. Executing any bytecode binaries found in bindir with with `-vnum` +6. Compilation and execution of a `print_endline Config.standard_library` + application with the fourteen compilation mechanisms (tests requiring the + shared runtime are skipped for `--disable-shared` or on Windows, where it is + isn't available and native mode tests are skipped for + `--disable-native-compiler`): + - Bytecode/native standard compilation + - Bytecode compilation with -custom running on the static/shared runtime + - Bytecode compilation with -output-complete-exe on the static/shared runtime + - Bytecode/native compilation with -output-obj linked with the static/shared + runtime + - Bytecode/native compilation with -output-complete-obj linked with the + static/shared runtime + +Having executed this battery on the configured prefix, the test then renames the +common root directory, appending the suffix `.new`. The programs compiled in the +sixth test are re-run and then the entire battery is executed a second time. + +During this second execution, the test harness does whatever is physically +possible to allow these tests to proceed: +- Environment variables `CAML_LD_LIBRARY_PATH` and `OCAMLLIB` are manipulated to + allow the compiler to operate +- Bytecode executables which will no longer be able to find `ocamlrun` are + explictly passed to `ocamlrun`. The harness always verifies that this step is + required by first executing the binary and ensuring that it fails and then + passing it directly to `ocamlrun`. + +## Tests + +In these descriptions, the Shims section describes the adjustments necessary for +the second phase of testing after the common root directory has been renamed. + +### Loading archives/plugins (.cma / .cmxa / .cmxs) in `ocaml` / `ocamlnat` + +Verifies that all the .cma files built from `otherlibs/` (dynlink, +runtime\_events, str, threads and unix) can be loaded in the two toplevels. +This test is skipped on builds which don't support shared libraries. + +Exercises: +- `CAML_LD_LIBRARY_PATH` and `ld.conf` logic (locating C stubs) +- ocamlnat's conversion machinery converting dynlink.cmxa to dynlink.cmxs +- `CAMLextern` header attribute for Windows (`RELOC_REL32`, etc.) + +Shims: +- On Unix, the bytecode toplevel contains the absolute location of `ocamlrun`, + so must be explicitly invoked via `ocamlrun` +- Both toplevels contain the absolute location of the Standard Library, + requiring `OCAMLLIB` to be set +- `ld.conf` contains the absolute location of the `stublibs` directory, + requiring `CAML_LD_LIBRARY_PATH` to be adjusted + +### Loading archives/plugins (.cma / .cmxs) with `Dynlink` + +As for the toplevel test, but using the `Dynlink` library. + +Shims: +- For a bytecode-only build, `ocamlc` contains the absolute location of + `ocamlrun`, so must be explicitly invoked via `ocamlrun` (if the native + compiler is available, then both `ocamlc` and `ocamlopt` will be native + executables) +- Both compilers contain the absolute location of the Standard Library, + requiring `OCAMLLIB` to be set +- The executable created by `ocamlc` contains the absolute location of + `ocamlrun`, so must be both explicitly invoked via `ocamlrun` and also have + `CAML_LD_LIBRARY_PATH` adjusted, as that `ocamlrun` will either not load + `ld.conf` or (with `OCAMLLIB` set) will be pointed to an `ld.conf` containing + the absolute location of the `stublibs` directory + +### Executing installed bytecode binaries with `-vnum` + +This test looks for filenames match `flexlink*` and `ocaml*` in bindir. For each +name matched, if the file ends with the bytecode magic number, then that program +is executed with `-vnum`. + +Exercises: +- Bytecode executable header and logic in `ocamlc` for computing the "shebang" + header + +Shims: +- On builds with shared library support, all the executables will contain the + absolute location of `ocamlrun` and will fail to execute +- On builds without shared library support, executables using libraries with + C stubs (in particular, `ocamldebug` and `ocamldoc`) are compiled with + `-custom` and do succeed +- The test ensures that executables fail exactly when expected, rather than + explicitly invoking them via `ocamlrun`. The reason for this is that some of + the binaries (in particular, `ocamlmktop`) invoke another executable (in + `ocamlmktop`'s case, `ocamlc`) which will itself fail if `ocamlc` is a + bytecode executable + +### Compilation mechanisms + +This battery of tests exercises each of the compilation mechanisms available in +OCaml with a simple test program linked with the `ocamlcommon` library which +displays the location of the Standard Library (as determined by +`Config.standard_library`) and checks whether that directory exists. + +Exercises: +- Default linking mode of both `ocamlc` ("tendered bytecode") and `ocamlopt` + (static executables) +- `-custom` and `-output-complete-exe` modes of `ocamlc` with both the default + static runtime and the shared runtime (`libcamlrun_shared`/`libasmrun_shared`) +- "Main program in C" linking modes `-output-obj` and `-output-complete-obj` + with both the static and shared runtimes + +Shims: +- As with the `Dynlink` test, on bytecode-only builds the compiler must be + explicitly invoked via `ocamlrun` +- The executable produced by `ocamlc` by default contains the absolute location + of `ocamlrun` and so has to be run explicitly via `ocamlrun` diff --git a/testsuite/tools/main_in_c.c b/testsuite/tools/main_in_c.c new file mode 100644 index 000000000000..9b034f5287da --- /dev/null +++ b/testsuite/tools/main_in_c.c @@ -0,0 +1,23 @@ +/**************************************************************************/ +/* */ +/* OCaml */ +/* */ +/* David Allsopp, Tarides */ +/* */ +/* Copyright 2024 David Allsopp Ltd. */ +/* */ +/* All rights reserved. This file is distributed under the terms of */ +/* the GNU Lesser General Public License version 2.1, with the */ +/* special exception on linking described in the file LICENSE. */ +/* */ +/**************************************************************************/ + +#define CAML_INTERNALS +#include + +int main_os(int argc, char_os **argv) +{ + caml_startup(argv); + caml_shutdown(); + return 0; +} diff --git a/testsuite/tools/test_in_prefix.ml b/testsuite/tools/test_in_prefix.ml new file mode 100644 index 000000000000..ae31ef769ef4 --- /dev/null +++ b/testsuite/tools/test_in_prefix.ml @@ -0,0 +1,1079 @@ +(**************************************************************************) +(* *) +(* OCaml *) +(* *) +(* David Allsopp, Tarides *) +(* *) +(* Copyright 2024 David Allsopp Ltd. *) +(* *) +(* All rights reserved. This file is distributed under the terms of *) +(* the GNU Lesser General Public License version 2.1, with the *) +(* special exception on linking described in the file LICENSE. *) +(* *) +(**************************************************************************) + +(* Parse command line. Result is the globally-immutable configuration and the + directories to use for bindir and libdir in both phases of the test. *) +type config = { + supports_shared_libraries: bool; + (* $(SUPPORTS_SHARED_LIBRARIES) - Makefile.config *) + has_ocamlnat: bool; + (* $(INSTALL_OCAMLNAT) - Makefile.build_config *) + has_ocamlopt: bool; + (* $(NATIVE_COMPILER) - Makefile.config *) + libraries: string list + (* Sorted basenames of libraries to test. + Derived from $(OTHERLIBRARIES) - Makefile.config *) +} + +(* bindir, libdir and config come from the command line. Validate that bindir + and libdir exist and share a common prefix (i.e. there is some prefix /foo + or C:\foo which they share) as otherwise it's not possible to rename the + installation directory. prefix is thus the common prefix of bindir and libdir + and [Filename.concat prefix bindir_suffix = bindir], etc. + *) +let bindir, libdir, prefix, bindir_suffix, libdir_suffix, + config, relocatable, target_relocatable = + (* Map directory names for otherlibs to library names and sort them in a + dependency-compatible order. *) + let sort_libraries libraries = + let compare l r = + if l = "unix" && r = "threads" then + -1 + else if l = "threads" && r = "unix" then + 1 + else + String.compare l r + in + List.map (function "systhreads" -> "threads" | lib -> lib) libraries + |> List.sort compare + in + let show_summary = ref false in + let bindir = ref "" in + let libdir = ref "" in + let config = + ref {supports_shared_libraries = false; + has_ocamlnat = false; has_ocamlopt = false; libraries = []} + in + let check_exists r dir = + if Sys.file_exists dir then + if Sys.is_directory dir then + if Filename.is_relative dir then + raise (Arg.Bad (dir ^ ": is not an absolute path")) + else + r := dir + else + raise (Arg.Bad (dir ^ ": not a directory")) + else + raise (Arg.Bad (dir ^ ": directory not found")) + in + let supports_shared_libraries supports_shared_libraries () = + config := {!config with supports_shared_libraries} + in + let has_ocamlnat has_ocamlnat () = config := {!config with has_ocamlnat} in + let has_ocamlopt has_ocamlopt () = config := {!config with has_ocamlopt} in + let args = Arg.align [ + "--bindir", Arg.String (check_exists bindir), "\ +\tDirectory containing programs (must share a prefix with --libdir)"; + "--libdir", Arg.String (check_exists libdir), "\ +\tDirectory containing stdlib.cma (must share a prefix with --bindir)"; + "--summary", Arg.Set show_summary, ""; + "--with-shared", Arg.Unit (supports_shared_libraries true), "\ +\tInstallation supports shared libraries (*.dll/*.so can be used from OCaml)"; + "--without-shared", Arg.Unit (supports_shared_libraries false), ""; + "--with-ocamlnat", Arg.Unit (has_ocamlnat true), "\ +\tNative toplevel (ocamlnat) is installed in the directory given in --bindir"; + "--without-ocamlnat", Arg.Unit (has_ocamlnat false), ""; + "--with-ocamlopt", Arg.Unit (has_ocamlopt true), "\ +\tNative compiler (ocamlopt) is installed in the directory given in --bindir"; + "--without-ocamlopt", Arg.Unit (has_ocamlopt false), ""; + ] in + let libraries lib = + config := {!config with libraries = lib::config.contents.libraries} + in + let usage = "\n\ +Usage: test_install --bindir --libdir [libraries]\n\ +options are:" in + let error fmt = + let f msg = + Printf.eprintf "%s: %s\n" Sys.executable_name msg; + if not !show_summary then + Arg.usage args usage; + exit 2 + in + Printf.ksprintf f fmt + in + let split_to_prefix bindir libdir = + let rec split_dir acc dir = + let dirname = Filename.dirname dir in + if dirname = dir then + dir::acc + else + split_dir (Filename.basename dir :: acc) dirname + in + let rec loop prefix bindir libdir = + match bindir, libdir with + | (dir1::bindir), (dir2::libdir) -> + if dir1 = dir2 then + loop (dir1::prefix) bindir libdir + else begin + match List.rev prefix with + | [] | [_] -> + (* The prefix is either the root directory (/, C:\, etc.) or, on + Windows, the two directories are actually on different drives + *) + error "\ +directories given for --bindir and --libdir do not have a common prefix"; + | dir::dirs -> + List.fold_left Filename.concat dir dirs, + List.fold_left Filename.concat dir1 bindir, + List.fold_left Filename.concat dir2 libdir + end + | [], _ -> + error "directory given for --libdir inside that given for --bindir" + | _, [] -> + error "directory given for --bindir inside that given for --libdir" + in + loop [] (split_dir [] bindir) (split_dir [] libdir) in + Arg.parse args libraries usage; + let config = + {!config with libraries = sort_libraries config.contents.libraries} + in + let {contents = bindir} = bindir in + let {contents = libdir} = libdir in + let relocatable = false in + let target_relocatable = false in + if bindir = "" || libdir = "" then + let () = Arg.usage args usage in + exit 2 + else + let prefix, bindir_suffix, libdir_suffix = split_to_prefix bindir libdir in + let style = + if Sys.getenv_opt "GITHUB_ACTIONS" <> None + || Sys.getenv_opt "APPVEYOR_BUILD_ID" <> None then + Some Misc.Color.Always + else + None + in + Misc.Style.setup style; + if Sys.file_exists (prefix ^ ".new") then + error "can't rename %s to %s.new as the latter already exists!" + prefix prefix; + let no_markup ansi = { Misc.Style.ansi; text_close = ""; text_open = "" } in + Misc.Style.(set_styles { + warning = no_markup [Bold; FG Yellow]; + error = no_markup [Bold; FG Red]; + loc = no_markup [Bold; FG Blue]; + hint = no_markup [Bold; FG Green]; + inline_code = no_markup [FG Blue]}); + let summary = + let choose b t f = (if b then t else f), true in + let puzzle = [ + "native and ", config.has_ocamlopt; + "bytecode", true; + " only", not config.has_ocamlopt; + " for ", true; + choose config.supports_shared_libraries + "shared and static linking" + "static linking only"; + " with ocamlnat", config.has_ocamlnat + ] in + let summary = + List.filter_map (fun (s, b) -> if b then Some s else None) puzzle + in + String.concat "" summary + in + if !show_summary then + exit 0; + Format.printf + "@{Test Environment@}\n\ + \ @{prefix@} = %s\n\ + \ @{bindir@} = [$prefix/]%s\n\ + \ @{libdir@} = [$prefix/]%s\n\ + Compiler is " prefix bindir_suffix libdir_suffix; + if relocatable then + Format.printf "@{relocatable@}; binaries produced are " + else + Format.printf "@{not relocatable@}; binaries produced are "; + if target_relocatable then + Format.printf "@{relocatable@}\n" + else + Format.printf "@{not relocatable@}\n"; + Format.printf "Testing %s\n%!" summary; + bindir, libdir, prefix, bindir_suffix, libdir_suffix, + config, relocatable, target_relocatable + +let test_root = Sys.getcwd () + +module Filename = struct + include Filename + + let is_dir_sep = + if Sys.win32 then + function '\\' | '/' -> true | _ -> false + else + (=) '/' +end + +module String = struct + include String + + let remove_prefix ~prefix s = + if starts_with ~prefix s then + let l = String.length prefix in + Some (String.sub s l (String.length s - l)) + else + None + + let find s p = + let max = length s - 1 in + if max = -1 then + None + else + let rec loop i = + if p s.[i] then + Some i + else if i < max then + loop (succ i) + else + None + in + loop 0 +end + +(* Jump through some mildly convoluted hoops to create diff'able output. + [display_path path] applies the following transformations: + - ["$bindir"] or ["$libdir"] if [path] is exactly [bindir_suffix] or + [libdir_suffix] (this captures passing those two variabes to the test + programs) + - if [path] begins with [prefix] then the text is replaced with ["$prefix"] + (which can create ["$prefix.new/"], etc.). Additionally, if the next part + of [path] after the following directory separator is [bindir_suffix] or + [libdir_suffix] then this is replaced with ["$bindir"] or ["$libdir"] + (i.e. this can generate ["$prefix.new/$bindir"] but not + ["$prefix.new/foo/$bindir"] + - if [path] begins [test_root] (i.e. the current directory) then this + is replaced with [Filename.current_dir_name] as long as [path] is either + exactly [test_root] or [test_root] is followed by a directory separator + (i.e. it generates ["./"] but never [".new/"]) + Both simpler and more convoluted ways of doing this are available. *) +let display_path f path = + match String.remove_prefix ~prefix path with + | Some remainder -> + if remainder = "" then + Format.pp_print_string f "$prefix" + else begin + match String.find remainder Filename.is_dir_sep with + | None -> + Format.fprintf f "$prefix%s" remainder + | Some idx -> + let suffix, path = + let idx = idx + 1 in + let suffix = String.sub remainder 0 idx in + let path = + String.sub remainder idx (String.length remainder - idx) + in + suffix, path + in + match String.remove_prefix ~prefix:bindir_suffix path with + | Some path when path = "" || Filename.is_dir_sep path.[0] -> + Format.fprintf f "$prefix%s$bindir%s" suffix path + | _ -> + match String.remove_prefix ~prefix:libdir_suffix path with + | Some path when path = "" || Filename.is_dir_sep path.[0] -> + Format.fprintf f "$prefix%s$libdir%s" suffix path + | _ -> + Format.pp_print_string f ("$prefix" ^ remainder) + end + | None -> + match String.remove_prefix ~prefix:test_root path with + | Some path when path = "" || Filename.is_dir_sep path.[0] -> + Format.pp_print_string f (Filename.current_dir_name ^ path) + | _ -> + if path = libdir_suffix then + Format.pp_print_string f "$libdir" + else if path = bindir_suffix then + Format.pp_print_string f "$bindir" + else + Format.pp_print_string f path + +type bytecode_classification = [ +| `Not_tendered_bytecode (* Produced by ocamlopt, -output-complete-exe, etc. *) +| `Shebang (* Launched using #! header *) +| `Launcher (* Launched using the executable launcher *) +| `Custom (* Compiled with -custom *) +] + +let classify_bytecode_image file : bytecode_classification = + try + In_channel.with_open_bin file (fun ic -> + let start = really_input_string ic 2 in + let is_RNTM = function + | Bytesections.{name = Name.RNTM; _} -> true + | _ -> false + in + let sections = Bytesections.(all (read_toc ic)) in + if start = "#!" then + `Shebang + else if List.exists is_RNTM sections then + `Launcher + else + `Custom) + with End_of_file | Bytesections.Bad_magic_number -> + `Not_tendered_bytecode + +let fail_because fmt = + let f s = + prerr_endline s; + exit 1 + in + Format.ksprintf f fmt + +type _ output = +| Stdout : unit output +| Return : (int * string list) output + +let print_process_status () = function +| Unix.WEXITED n -> "exited with " ^ string_of_int n +| Unix.WSIGNALED _ -> "signalled" +| Unix.WSTOPPED _ -> "stopped" + +module Environment : sig + type t + + val make : + ?caml_ld_library_path:bool -> ?ocamllib:bool -> string -> string -> t + + val run_process : + 'a output + -> ?runtime:string -> string -> string list -> ?no_stderr:bool -> t -> 'a + + val run_process_target : + 'a output + -> ?runtime:string -> string -> string list -> ?no_stderr:bool -> t -> 'a +end = struct + type t = { + env: string array; + serial: int; + bindir: string; + libdir: string; + set_CAML_LD_LIBRARY_PATH: bool; + set_OCAMLLIB: bool; + } + + module StringSet = Set.Make(String) + + (* List of environment variables to remove from the calling environment *) + let scrub = + StringSet.of_list [ + "BUILD_PATH_PREFIX_MAP"; + "CAMLLIB"; + "CAMLRUNPARAM"; + "CAML_LD_LIBRARY_PATH"; + "OCAMLLIB"; + "OCAMLPARAM"; + "OCAMLRUNPARAM"; + "OCAMLTOP_INCLUDE_PATH"; + "OCAML_RUNTIME_EVENTS_DIR"; + "OCAML_RUNTIME_EVENTS_PRESERVE"; + "OCAML_RUNTIME_EVENTS_START"; + ] + + (* Tests whether the name of an environment variable is in fact PATH, + masking the fact that environment variable names are case-insensitive on + Windows. *) + let is_path_env = + if Sys.win32 then + fun name -> String.lowercase_ascii name = "path" + else + String.equal "PATH" + + let environments = Hashtbl.create 15 + + let ld_library_path_name = + if Config.system = "macosx" then + "DYLD_LIBRARY_PATH" + else + "LD_LIBRARY_PATH" + + (* Returns an environment where any variables in scrub have been removed and + with effectively PATH=$bindir:$PATH and + LD_LIBRARY_PATH=$libdir:$LD_LIBRARY_PATH on Unix or + DYLD_LIBRARY_PATH=$libdir$:DYLD_LIBRARY_PATH on macOS or + PATH=$bindir;$libdir;$PATH on Windows. *) + let make ?(caml_ld_library_path=false) ?(ocamllib=false) bindir libdir = + let keep binding = + let equals = String.index binding '=' in + let name = String.sub binding 0 equals in + let value = + String.sub binding (equals + 1) (String.length binding - equals - 1) + in + if StringSet.mem name scrub then + None + else if is_path_env name then + if Sys.win32 then + if String.index_opt bindir ';' <> None then + Some (Printf.sprintf "%s=\"%s\";%s" name bindir value) + else + Some (Printf.sprintf "%s=%s;%s" name bindir value) + else + Some (Printf.sprintf "%s=%s:%s" name bindir value) + else if not Sys.win32 && name = ld_library_path_name then + Some (Printf.sprintf "%s=%s:%s" name libdir value) + else + Some binding + in + let bindings = List.filter_map keep (Array.to_list (Unix.environment ())) in + let bindings = + if Sys.win32 + || List.exists (String.starts_with ~prefix:(ld_library_path_name ^ "=")) + bindings then + bindings + else + (ld_library_path_name ^ "=" ^ libdir)::bindings + in + let bindings = + if ocamllib then begin + assert (not relocatable); + ("OCAMLLIB=" ^ libdir)::bindings + end else + bindings + in + let bindings = + if caml_ld_library_path then begin + assert (not relocatable); + ("CAML_LD_LIBRARY_PATH=" ^ Filename.concat libdir "stublibs")::bindings + end else + bindings + in + let env = Array.of_list bindings in + try {env; serial = Hashtbl.find environments env; bindir; libdir; + set_CAML_LD_LIBRARY_PATH = caml_ld_library_path; + set_OCAMLLIB = ocamllib} + with Not_found -> + let serial = Hashtbl.length environments + 1 in + Hashtbl.add environments env serial; + {env; serial; bindir; libdir; + set_CAML_LD_LIBRARY_PATH = caml_ld_library_path; set_OCAMLLIB = ocamllib} + + let null = Unix.openfile Filename.null [Unix.O_WRONLY] 0o200 + + let last_environment = ref (-1) + + (* [run_process output_mode ?runtime program args ?no_stderr env] executes + [program] with [args] (note that [args] does not need to include [program] + at its head). If [no_stderr = true], then standard error is suppressed, + otherwise anything output on standard error is interleaved with standard + output. If [runtime] is given, then it is the full path to ocamlrun. In + this case, if program is a bytecode image, then program is expected to fail + (either with ENOENT, for a #!-style launcher or with exit code 2) and + [run_process] retries, explicitly passing [program] to [runtime]. If + [output_mode] is [Stdout], then ultimately the process must terminate with + exit code 0, or the test harness aborts. If [output_mode] is [Return], then + the final exit code is returned, along with the lines of output from the + process. + Both the processes potentially launched by this function are displayed on + the console. If the environment is different from the last time + [run_process] was called then a summary of the changes in it are also + displayed. *) + let rec run_process_aux ~to_stdout ?runtime program args ?(no_stderr = false) + ({env; _} as environment) = + flush stderr; + flush stdout; + let captured_output = "process-output" in + let stdout, stderr = + let flags = Unix.([O_RDWR; O_CREAT; O_TRUNC; O_CLOEXEC]) in + let fd = Unix.openfile captured_output flags 0o600 in + fd, fd + in + let stderr = if no_stderr || runtime <> None then null else stderr in + let runtime, classification = + match runtime with + | None -> + "", `Not_tendered_bytecode + | Some runtime -> + let classification = classify_bytecode_image program in + let runtime = + if classification = `Custom then + "" + else + runtime + in + runtime, classification + in + let summarise f () = + display_path f program; + List.iter (fun x -> Format.pp_print_char f ' '; display_path f x) args; + if not to_stdout then + Format.pp_print_string f (" 1> " ^ captured_output); + if no_stderr then + Format.pp_print_string f (" 2>" ^ Filename.null) + in + let display_environment () = + if environment.serial <> !last_environment then begin + last_environment := environment.serial; + (* For ease of diff'ing, the environment is displayed in Posix format + and ignores the fact Windows doesn't set LD_LIBRARY_PATH *) + Format.printf "\ +@{> @}@{Environment@}\n\ +@{> @} @{PATH@}=%a:$PATH\n\ +@{> @} @{LD_LIBRARY_PATH@}=%a:$LD_LIBRARY_PATH\n" + display_path environment.bindir + display_path environment.libdir; + if environment.set_CAML_LD_LIBRARY_PATH then + Format.printf "\ +@{> @} @{CAML_LD_LIBRARY_PATH@}=\ + %a/stublibs:$CAML_LD_LIBRARY_PATH\n" + display_path libdir; + if environment.set_OCAMLLIB then + Format.printf "\ +@{> @} @{OCAMLLIB@}=%a:$OCAMLLIB\n" display_path libdir + end + in + try + let pid = + Unix.create_process_env program (Array.of_list (program::args)) env + Unix.stdin stdout stderr + in + let (_, status) = Unix.waitpid [] pid in + begin + match status with + | Unix.WEXITED n -> + if n = 2 && (not to_stdout || classification = `Launcher) then + Format.printf "@{%a@} <@{exit 2@}>\n%!" + summarise () + else if n = 0 then + Format.printf "@{%a@}\n%!" summarise () + else + Format.printf "@{%a@} <@{exit %d@}>\n%!" + summarise () n + | _ -> + Format.printf "@{%a@}\n%!" summarise () + end; + display_environment (); + let result = + let _ = Unix.lseek stdout 0 Unix.SEEK_SET in + let lines = + let ic = Unix.in_channel_of_descr stdout in + In_channel.set_binary_mode ic false; + if to_stdout then + let format_line () = Format.printf "@{>@} %s\n%!" in + let () = In_channel.fold_lines format_line () ic in + [] + else + In_channel.input_lines ic + in + Unix.close stdout; + Sys.remove captured_output; + lines in + match status with + | Unix.WEXITED n when runtime = "" && (n = 0 || not to_stdout) -> + n, result + | Unix.WEXITED 2 when classification = `Launcher -> + run_process_aux ~to_stdout runtime (program::args) ~no_stderr + environment + | status -> + fail_because "%s did not terminate as expected (%a)" + program print_process_status status + with Unix.(Unix_error(ENOENT, "create_process", _)) as e -> + Format.printf "@{%a@} <@{exit 2@}>\n%!" summarise (); + display_environment (); + if classification = `Shebang || classification = `Launcher then + run_process_aux ~to_stdout runtime (program::args) ~no_stderr + environment + else if to_stdout then + raise e + else + (2, []) + + let run_process : type s . guard:bool -> s output -> ?runtime:string -> string + -> string list -> ?no_stderr:bool -> t -> s = + fun ~guard output ?runtime program args ?no_stderr env -> + assert (runtime = None || not guard); + match output with + | Stdout -> + run_process_aux ~to_stdout:true ?runtime program args ?no_stderr env + |> ignore + | Return -> + run_process_aux ~to_stdout:false ?runtime program args ?no_stderr env + + let run_process_target output = run_process ~guard:target_relocatable output + let run_process output = run_process ~guard:relocatable output +end + +(* When compiling with -output-obj, any additional C flags specified in + ocamlcommon.cmxa will be omitted and must be specified when linking. Read the + C lib_ccobjs field from ocamlcommon.cmxa for these tests. *) +let ocamlcommon_native_c_libraries = + if config.has_ocamlopt then + let lib = + Filename.(concat (concat libdir "compiler-libs") "ocamlcommon.cmxa") + in + In_channel.with_open_bin lib (fun ic -> + try + let magic = + really_input_string ic (String.length Config.cmxa_magic_number) + in + if magic <> Config.cmxa_magic_number then + fail_because "Wrong magic number in %s; expected %s but got %s" + lib Config.cmxa_magic_number magic; + let {Cmx_format.lib_ccobjs = libs; lib_ccopts = opts} = + (input_value ic : Cmx_format.library_infos) + in + if libs <> [] then + String.concat " " (List.rev (libs @ opts)) + else + "" + with End_of_file -> + fail_because "%s appears to be corrupted" lib) + else + "" + +(* exe ["foo" = "foo.exe"] on Windows or ["foo"] otherwise. *) +let exe = + if Sys.win32 then + Fun.flip (^) ".exe" + else + Fun.id + +type mode = Bytecode | Native + +(* This test verifies that a series of libraries can be loaded in a toplevel. + Any failures cause the script to be aborted. *) +let load_libraries_in_toplevel ~original env bindir mode libraries = + let toplevel = + match mode with + | Bytecode -> "ocaml" + | Native -> "ocamlnat" + in + let toplevel = Filename.concat bindir (exe toplevel) in + Format.printf "\nTesting loading of libraries in %a\n%!" + display_path toplevel; + Out_channel.with_open_text "test_install_script.ml" (fun oc -> + List.iter (fun library -> + (* dynlink.cmxs does not exist, for obvious reasons, but we can check + loading the library in ocamlnat "works". *) + let ext = + match mode with + | Native -> + if library = "dynlink" then + "cmxa" + else + "cmxs" + | Bytecode -> + "cma" + in + Printf.fprintf oc + "#directory \"+%s\";;\n\ + #load \"%s.%s\";;\n\ + print_endline \"Loaded %s.%s\";;" + library library ext library ext) libraries; + Printf.fprintf oc "#quit;;\n"); + let args = + ["-noinit"; "-no-version"; "-noprompt"; "test_install_script.ml"] + in + let runtime = + if original || Sys.win32 || mode = Native then + None + else + Some (Filename.concat bindir (exe "ocamlrun")) + in + Environment.run_process Stdout ?runtime toplevel args env; + Sys.remove "test_install_script.ml" + +(* This test verifies that a series of libraries can be loaded via Dynlink. + Any failures will cause either an exception or a compilation error. *) +let load_libraries_in_prog ~original env bindir libdir mode libraries = + Format.printf "\nTesting loading of libraries with %s dynlink\n" + (if mode = Native then "native" else "bytecode"); + let ocamlrun = Some (Filename.concat bindir (exe "ocamlrun")) in + Out_channel.with_open_text "test_install_script.ml" (fun oc -> + let emit_library library = + if library <> "dynlink" then + let libdir = Filename.concat libdir library in + let library = library ^ ".cma" in + let lib = Filename.concat libdir library in + Printf.fprintf oc + " Dynlink.loadfile (Dynlink.adapt_filename %S);\n\ + \ Printf.printf \"Loaded %%s\\n\" (Dynlink.adapt_filename %S);" + lib library + + in + Printf.fprintf oc + "let () =\n\ + \ let () = Dynlink.allow_unsafe_modules true in\n"; + List.iter emit_library libraries; + ); + flush stdout; + let test_program = Filename.concat test_root (exe "test_install_script") in + let () = + let compiler, dynlink = + match mode with + | Bytecode -> "ocamlc", "dynlink.cma" + | Native -> "ocamlopt", "dynlink.cmxa" + in + let compiler = Filename.concat bindir (exe compiler) in + let runtime = + if original || Sys.win32 || config.has_ocamlopt || mode = Native then + None + else + ocamlrun + in + let args = [ + "-I"; "+dynlink"; dynlink; "-linkall"; + "-o"; test_program; "test_install_script.ml" + ] in + Environment.run_process Stdout ?runtime compiler args env + in + let runtime = + if original || Sys.win32 || mode = Native then + None + else + ocamlrun + in + let () = Environment.run_process_target Stdout ?runtime test_program [] env in + let files = [ + test_program; + "test_install_script.ml"; + "test_install_script.cmi"; + "test_install_script.cm" ^ (if mode = Native then "x" else "o") + ] in + let files = + if mode = Native then + ("test_install_script" ^ Config.ext_obj)::files + else + files + in + List.iter Sys.remove files + +let is_executable = + if Sys.win32 then + Fun.const true + else + fun binary -> + try Unix.access binary [Unix.X_OK]; true + with Unix.Unix_error _ -> false + +(* This test verifies that a series of libraries can be loaded via Dynlink. + Any failures will cause either an exception or a compilation error. *) +let test_bytecode_binaries ~original env bindir = + let test_binary binary = + if String.starts_with ~prefix:"ocaml" binary + || String.starts_with ~prefix:"flexlink" binary then + let binary = Filename.concat bindir binary in + if is_executable binary then + match classify_bytecode_image binary with + | `Not_tendered_bytecode -> () + | `Shebang | `Launcher | `Custom -> + match Environment.run_process Return binary ["-vnum"] env with + | (0, output) -> + let format_line = Format.printf "@{>@} %s\n%!" in + List.iter format_line output + | (2, _) when not original && not Sys.win32 -> + () + | _ -> + fail_because "it was broken" + in + let binaries = Sys.readdir bindir in + Format.printf "\nTesting bytecode binaries in %a\n" display_path bindir; + Array.sort String.compare binaries; + Array.iter test_binary binaries + +let write_test_program description = + Out_channel.with_open_text "test_install_script.ml" (fun oc -> + Printf.fprintf oc {| +let state = bool_of_string Sys.argv.(1) + +let is_directory dir = + try Sys.is_directory dir + with Sys_error _ -> false + +let display_lib = + let dir = Config.standard_library in + let dir = + if String.starts_with ~prefix:Sys.argv.(2) dir then + let l = String.length Sys.argv.(2) in + "$prefix" ^ String.sub dir l (String.length dir - l) + else + dir + in + if String.ends_with ~suffix:Sys.argv.(3) dir then + let l = String.length Sys.argv.(3) in + String.sub dir 0 (String.length dir - l) ^ "$libdir" + else + dir + +let () = + Printf.printf "%s: %%s\n%%!" display_lib; + if is_directory Config.standard_library <> state then + let () = + Printf.eprintf " *** Directory %%sfound!\n" + (if state then "not " else "") + in + exit 1 +|} description) + +let run_program env ?runtime test_program ~arg = + let args = [string_of_bool arg; prefix; libdir_suffix] in + Environment.run_process_target Stdout ?runtime test_program args env + +type compiler = C_ocamlc | C_ocamlopt +type runtime_mode = Shared | Static +type linkage = +| Default of compiler +| Custom of runtime_mode +| Output_obj of compiler * runtime_mode +| Output_complete_obj of compiler * runtime_mode +| Output_complete_exe of runtime_mode + +let compile_test ~original env bindir = + let runtime = + if original || config.has_ocamlopt || Sys.win32 then + None + else + Some (Filename.concat bindir (exe "ocamlrun")) + in + let ocamlc = Filename.concat bindir (exe "ocamlc") in + let ocamlopt = Filename.concat bindir (exe "ocamlopt") in + let main_object = + Filename.concat (Filename.dirname Sys.executable_name) + ("main_in_c" ^ Config.ext_obj) + in + fun test test_program description -> + let use_shared_runtime, needs_ocamlopt, options, main_in_c, clibs = + match test with + | Default C_ocamlc -> + false, false, [], false, None + | Default C_ocamlopt -> + false, true, [], false, None + | Custom Static -> + false, false, ["-custom"], false, None + | Custom Shared -> + true, false, ["-custom"], false, None + | Output_obj(C_ocamlc, Static) -> + false, false, ["-output-obj"], true, None + | Output_obj(C_ocamlc, Shared) -> + true, false, ["-output-obj"], true, None + | Output_obj(C_ocamlopt, Static) -> + false, true, ["-output-obj"], true, + Some ocamlcommon_native_c_libraries + | Output_obj(C_ocamlopt, Shared) -> + true, true, ["-output-obj"], true, + Some ocamlcommon_native_c_libraries + | Output_complete_obj(C_ocamlc, Static) -> + false, false, ["-output-complete-obj"], true, None + | Output_complete_obj(C_ocamlc, Shared) -> + true, false, ["-output-complete-obj"], true, None + | Output_complete_obj(C_ocamlopt, Static) -> + (* At the moment, the partial linker will pass -lzstd to ld -r which + will (normally) fail). Until this is done, pass the libraries + manually. *) + false, true, ["-output-complete-obj"; "-noautolink"], true, + Some ocamlcommon_native_c_libraries + | Output_complete_obj(C_ocamlopt, Shared) -> + true, true, ["-output-complete-obj"; "-noautolink"], true, + Some ocamlcommon_native_c_libraries + | Output_complete_exe Static -> + false, false, ["-output-complete-exe"], false, None + | Output_complete_exe Shared -> + true, false, ["-output-complete-exe"], false, None + in + (* At present, shared runtime support is not available on Windows *) + if use_shared_runtime + && (Sys.win32 || not config.supports_shared_libraries) + || needs_ocamlopt && not config.has_ocamlopt then + None + else + let test_program = Filename.concat test_root (exe test_program) in + let compiler = if needs_ocamlopt then ocamlopt else ocamlc in + let compile_with_main_in_c output = + let runtime_lib = + let suffix = if use_shared_runtime then "_shared" else "" in + if needs_ocamlopt then + "-lasmrun" ^ suffix + else + "-lcamlrun" ^ suffix + in + let flags = + let libraries = + if needs_ocamlopt then + [runtime_lib; Config.native_c_libraries] + else + [runtime_lib; Config.bytecomp_c_libraries] + in + let libraries = + match clibs with + | Some lib when lib <> "" -> lib::libraries + | _ -> libraries + in + String.concat " " libraries + in + if Ccomp.call_linker Ccomp.Exe test_program + [output; main_object] flags <> 0 then + fail_because "Unexpected linker error"; + Sys.remove output + in + let () = + let output = + if main_in_c then + "test_install_ocaml" ^ Config.ext_obj + else + test_program + in + write_test_program description; + let ocamlcommon = + if needs_ocamlopt then "ocamlcommon.cmxa" else "ocamlcommon.cma" + in + let options = + if use_shared_runtime && not main_in_c then + "-runtime-variant" :: "_shared" :: options + else + options + in + let args = + "-I" :: "+compiler-libs" :: ocamlcommon :: + "-o" :: output :: + "test_install_script.ml" :: options + in + let () = Environment.run_process Stdout ?runtime compiler args env in + let files = [ + "test_install_script.ml"; + "test_install_script.cmi"; + "test_install_script.cm" ^ (if needs_ocamlopt then "x" else "o") + ] in + let files = + if needs_ocamlopt then + ("test_install_script" ^ Config.ext_obj)::files + else + files + in + List.iter Sys.remove files; + if main_in_c then + compile_with_main_in_c output + in + let rec run ~original ?runtime env ~arg = + let runtime = + if test = Default C_ocamlc && not Sys.win32 then + runtime + else + None + in + run_program env ?runtime test_program ~arg; + if original then + Some (run ~original:false) + else + (Sys.remove test_program; None) + in + Some (run ~original) + +let compiler_where env ?runtime compiler = + match Environment.run_process Return ?runtime compiler ["-where"] env with + | (0, [where]) -> where + | _ -> + fail_because "Unexpected response from %s -where" compiler + +(* This test verifies both that all compilation mechanisms are working and that + each of these programs can correctly identify the Standard Library location. + Any failures will cause either an exception or a compilation error. *) +let test_standard_library_location ~original env bindir = + Format.printf "\nTesting compilation mechanisms for %a\n%!" + display_path bindir; + let runtime = + if original || config.has_ocamlopt || Sys.win32 then + None + else + Some (Filename.concat bindir (exe "ocamlrun")) + in + let ocamlc = Filename.concat bindir (exe "ocamlc") in + let ocamlopt = Filename.concat bindir (exe "ocamlopt") in + let ocamlc_where = compiler_where env ?runtime ocamlc in + let ocamlopt_where = + if config.has_ocamlopt then + compiler_where env ocamlopt + else + "n/a" + in + Format.printf "ocamlc -where: %a\nocamlopt -where: %a\n%!" + display_path ocamlc_where display_path ocamlopt_where; + let compile_test = compile_test ~original env bindir in + let programs = List.filter_map Fun.id [ + compile_test (Default C_ocamlc) + "byt_default" "with tender"; + compile_test (Custom Static) + "custom_static" "-custom static runtime"; + compile_test (Custom Shared) + "custom_shared" "-custom shared runtime"; + compile_test (Output_obj(C_ocamlc, Static)) + "byt_obj_static" "-output-obj static runtime"; + compile_test (Output_obj(C_ocamlc, Shared)) + "byt_obj_shared" "-output-obj shared runtime"; + compile_test (Output_complete_obj(C_ocamlc, Static)) + "byt_complete_obj_static" "-output-complete-obj static runtime"; + compile_test (Output_complete_obj(C_ocamlc, Shared)) + "byt_complete_obj_shared" "-output-complete-obj shared runtime"; + compile_test (Output_complete_exe Static) + "byt_complete_exe_static" "-output-complete-exe static runtime"; + compile_test (Output_complete_exe Shared) + "byt_complete_exe_shared" "-output-complete-exe shared runtime"; + compile_test (Default C_ocamlopt) + "nat_default" "static runtime"; + compile_test (Output_obj(C_ocamlopt, Static)) + "nat_obj_static" "-output-obj static runtime"; + compile_test (Output_obj(C_ocamlopt, Shared)) + "nat_obj_shared" "-output-obj shared runtime"; + compile_test (Output_complete_obj(C_ocamlopt, Static)) + "nat_complete_obj_static" "-output-complete-obj static runtime"; + compile_test (Output_complete_obj(C_ocamlopt, Shared)) + "nat_complete_obj_shared" "-output-complete-obj shared runtime"; + ] in + let runtime = + if original then + None + else + Some (Filename.concat bindir (exe "ocamlrun")) + in + Printf.printf "Running programs\n%!"; + List.filter_map (fun f -> f ?runtime env ~arg:true) programs + +let run_tests ~original env bindir libdir libraries = + if config.supports_shared_libraries then + load_libraries_in_toplevel ~original env bindir Bytecode libraries; + if config.has_ocamlnat then + load_libraries_in_toplevel ~original env bindir Native libraries; + if config.supports_shared_libraries then + load_libraries_in_prog ~original env bindir libdir Bytecode libraries; + if config.has_ocamlopt && config.supports_shared_libraries then + load_libraries_in_prog ~original env bindir libdir Native libraries; + test_bytecode_binaries ~original env bindir; + test_standard_library_location ~original env bindir + +let () = + (* Run all tests in the supplied prefix *) + Compmisc.init_path (); + let env = Environment.make bindir libdir in + let programs = run_tests ~original:true env bindir libdir config.libraries in + (* Now rename the prefix, appending .new to the directory name *) + let new_prefix = prefix ^ ".new" in + let bindir = Filename.concat new_prefix bindir_suffix in + let libdir = Filename.concat new_prefix libdir_suffix in + Format.printf "\nRenaming %a to %a\n\n%!" display_path prefix + display_path new_prefix; + Sys.rename prefix new_prefix; + at_exit (fun () -> + flush stderr; + flush stdout; + Format.printf "\nRestoring %a to %a\n" display_path new_prefix + display_path prefix; + Sys.rename new_prefix prefix); + (* Re-run the test programs compiled with the normal prefix *) + Printf.printf "Re-running test programs\n%!"; + (* Finally re-run all of the tests with the new prefix *) + let env = Environment.make bindir libdir in + let runtime = Some (Filename.concat bindir (exe "ocamlrun")) in + List.iter (fun f -> assert (f ?runtime env ~arg:false = None)) programs; + let env = + Environment.make ~caml_ld_library_path:true ~ocamllib:true bindir libdir + in + Compmisc.reinit_path ~standard_library:libdir (); + let programs = run_tests ~original:false env bindir libdir config.libraries in + assert (programs = []) diff --git a/testsuite/tools/test_in_prefix.mli b/testsuite/tools/test_in_prefix.mli new file mode 100644 index 000000000000..fb019d3cc886 --- /dev/null +++ b/testsuite/tools/test_in_prefix.mli @@ -0,0 +1,13 @@ +(**************************************************************************) +(* *) +(* OCaml *) +(* *) +(* David Allsopp, Tarides *) +(* *) +(* Copyright 2024 David Allsopp Ltd. *) +(* *) +(* All rights reserved. This file is distributed under the terms of *) +(* the GNU Lesser General Public License version 2.1, with the *) +(* special exception on linking described in the file LICENSE. *) +(* *) +(**************************************************************************) From 4a6c8276fd8043e850301b8b803926b501a9bb4a Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Fri, 5 Jul 2024 09:39:51 +0100 Subject: [PATCH 4/8] Add missing --enable-ocamltest to MSVC CI Necessary when building release tags. --- .github/workflows/build-msvc.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-msvc.yml b/.github/workflows/build-msvc.yml index 960e6f24da06..39c3e1ece297 100644 --- a/.github/workflows/build-msvc.yml +++ b/.github/workflows/build-msvc.yml @@ -94,12 +94,17 @@ jobs: env: HOST: ${{ matrix.x86_64 && 'x86_64-pc-windows' || 'i686-pc-windows' }} CC: ${{ matrix.cc }} + CONFIGURE_FLAGS: >- + --cache-file=config.cache + --host=$HOST + CC=$CC + --enable-ocamltest run: >- eval $(tools/msvs-promote-path) ; - if ! ./configure --cache-file=config.cache --host=$HOST CC=$CC ; then + if ! ./configure ${{ env.CONFIGURE_FLAGS }} ; then rm -rf config.cache ; failed=0 ; - ./configure --cache-file=config.cache --host=$HOST CC=$CC \ + ./configure ${{ env.CONFIGURE_FLAGS }} \ || failed=$?; if ((failed)) ; then cat config.log ; exit $failed ; fi ; fi ; From b9cf4a4da15cb1ff8b421fe66b52b95c49e9b6ee Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Fri, 5 Jul 2024 09:27:50 +0100 Subject: [PATCH 5/8] Plumb the installation tests into CI --- .github/workflows/build-msvc.yml | 13 +++++++++++++ .github/workflows/build.yml | 27 ++++++++++++++++++++++++--- tools/ci/actions/runner.sh | 6 ++++++ tools/ci/appveyor/appveyor_build.sh | 7 ++++++- tools/ci/inria/main | 1 + 5 files changed, 50 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-msvc.yml b/.github/workflows/build-msvc.yml index 39c3e1ece297..b59fb777366a 100644 --- a/.github/workflows/build-msvc.yml +++ b/.github/workflows/build-msvc.yml @@ -96,9 +96,11 @@ jobs: CC: ${{ matrix.cc }} CONFIGURE_FLAGS: >- --cache-file=config.cache + --prefix "$PWD/install" --host=$HOST CC=$CC --enable-ocamltest + ${{ matrix.x86_64 && '--enable-native-toplevel' || '--disable-native-toplevel' }} run: >- eval $(tools/msvs-promote-path) ; if ! ./configure ${{ env.CONFIGURE_FLAGS }} ; then @@ -142,3 +144,14 @@ jobs: run: >- eval $(tools/msvs-promote-path) ; make -j tests ; + + - name: Install + shell: bash + run: >- + make install + + - name: Test in prefix + shell: bash + run: >- + eval $(tools/msvs-promote-path) ; + make -f Makefile.test -C testsuite/in_prefix test-in-prefix diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3f7ea758c57..95be98cbba6a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,6 +136,10 @@ jobs: if: matrix.id == 'normal' run: | MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh install + - name: Test in prefix + if: matrix.id == 'normal' + run: | + MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh test-in-prefix - name: Build the manual if: matrix.id == 'normal' && needs.build.outputs.manual_changed == 'true' run: | @@ -157,14 +161,17 @@ jobs: config_arg: CFLAGS='-O0' - name: macos-x86_64 os: macos-13 + in-prefix-test: true - name: macos-arm64 os: macos-14 + in-prefix-test: true - name: static os: ubuntu-latest - config_arg: --disable-shared + config_arg: --disable-shared --disable-native-toplevel + in-prefix-test: true - name: minimal os: ubuntu-latest - config_arg: --disable-native-compiler --disable-shared --disable-debug-runtime --disable-instrumented-runtime --disable-systhreads --disable-str-lib --disable-unix-lib --disable-ocamldoc + config_arg: --disable-native-compiler --disable-native-toplevel --disable-shared --disable-debug-runtime --disable-instrumented-runtime --disable-systhreads --disable-str-lib --disable-unix-lib --disable-ocamldoc fail-fast: false steps: - name: Checkout @@ -215,6 +222,14 @@ jobs: for dir in $PARALLEL_TESTS; do \ bash -cxe "SHOW_TIMINGS=1 tools/ci/actions/runner.sh test_prefix $dir"; \ done + - name: Install + if: ${{ matrix.in-prefix-test }} + run: | + MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh install + - name: Test in prefix + if: ${{ matrix.in-prefix-test }} + run: | + MAKE_ARG=-j OCAMLRUNPARAM=b,v=0 bash -xe tools/ci/actions/runner.sh test-in-prefix i386: runs-on: ubuntu-latest @@ -233,10 +248,16 @@ jobs: - name: configure tree run: | chown -R ocaml:ocaml . - MAKE_ARG=-j su ocaml -c "bash -xe tools/ci/actions/runner.sh configure" + MAKE_ARG=-j CONFIG_ARG=--disable-native-toplevel su ocaml -c "bash -xe tools/ci/actions/runner.sh configure" - name: Build run: | MAKE_ARG=-j su ocaml -c "bash -xe tools/ci/actions/runner.sh build" - name: Run the testsuite run: | su ocaml -c "bash -xe tools/ci/actions/runner.sh test" + - name: Install + run: | + su ocaml -c "bash -xe tools/ci/actions/runner.sh install" + - name: Test in prefix + run: | + su ocaml -c "bash -xe tools/ci/actions/runner.sh test-in-prefix" diff --git a/tools/ci/actions/runner.sh b/tools/ci/actions/runner.sh index b78b53f8a319..860832e7f8ac 100755 --- a/tools/ci/actions/runner.sh +++ b/tools/ci/actions/runner.sh @@ -41,6 +41,7 @@ EOF --prefix=$PREFIX \ --enable-flambda-invariants \ --enable-ocamltest \ + --enable-native-toplevel \ --disable-dependency-generation \ $CONFIG_ARG" @@ -114,6 +115,10 @@ Install () { $MAKE install } +Test-In-Prefix () { + $MAKE -C testsuite/in_prefix -f Makefile.test test-in-prefix +} + Checks () { if fgrep 'SUPPORTS_SHARED_LIBRARIES=true' Makefile.config &>/dev/null ; then echo Check the code examples in the manual @@ -209,6 +214,7 @@ test_sequential) Test sequential;; test_prefix) TestPrefix $2;; api-docs) API_Docs;; install) Install;; +test-in-prefix) Test-In-Prefix;; manual) BuildManual;; other-checks) Checks;; basic-compiler) BasicCompiler;; diff --git a/tools/ci/appveyor/appveyor_build.sh b/tools/ci/appveyor/appveyor_build.sh index ea43236b6a7f..304248ce909c 100755 --- a/tools/ci/appveyor/appveyor_build.sh +++ b/tools/ci/appveyor/appveyor_build.sh @@ -95,11 +95,13 @@ function set_configuration { # Remove configure cache if the script has failed if ! ./configure --cache-file="$CACHE_FILE" $dep $build $man $host \ - --prefix="$2" --enable-ocamltest ; then + --prefix="$2" --enable-ocamltest \ + --enable-native-toplevel ; then rm -f -- "$CACHE_FILE" local failed ./configure --cache-file="$CACHE_FILE" $dep $build $man $host \ --prefix="$2" --enable-ocamltest \ + --enable-native-toplevel \ || failed=$? if ((failed)) ; then cat config.log ; exit $failed ; fi fi @@ -173,6 +175,9 @@ case "$1" in run "test $PORT" \ make -C "$FULL_BUILD_PREFIX-$PORT/testsuite" SHOW_TIMINGS=1 all run "install $PORT" $MAKE -C "$FULL_BUILD_PREFIX-$PORT" install + run "test installation of $PORT" \ + $MAKE -f Makefile.test -C "$FULL_BUILD_PREFIX-$PORT/testsuite/in_prefix" \ + test-in-prefix if [[ $PORT = 'msvc64' ]] ; then run "$MAKE check_all_arches" \ $MAKE -C "$FULL_BUILD_PREFIX-$PORT" check_all_arches diff --git a/tools/ci/inria/main b/tools/ci/inria/main index 5ece025cb4fc..1f6238e36802 100755 --- a/tools/ci/inria/main +++ b/tools/ci/inria/main @@ -311,6 +311,7 @@ if $make_native && $check_make_alldepend; then fi $make --warn-undefined-variables install +$make -f Makefile.test -C testsuite/in_prefix test-in-prefix rm -rf "$instdir" cd testsuite From aeaabe19b32ff112970284f28b2967d7afe4f271 Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Sat, 6 Jul 2024 09:27:03 +0100 Subject: [PATCH 6/8] Only call Cygwin setup once If upgrading is forced and packages are required, combine the two calls. --- tools/ci/appveyor/appveyor_build.cmd | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/ci/appveyor/appveyor_build.cmd b/tools/ci/appveyor/appveyor_build.cmd index c49027a3d70d..6f7a2bd17dec 100644 --- a/tools/ci/appveyor/appveyor_build.cmd +++ b/tools/ci/appveyor/appveyor_build.cmd @@ -61,11 +61,12 @@ if %ERRORLEVEL% equ 1 ( goto :EOF :UpgradeCygwin +set CYGWIN_FLAGS= if %CYGWIN_UPGRADE_REQUIRED% equ 1 ( - set CYGWIN_FLAGS=--upgrade-also - set CYGWIN_UPGRADE_REQUIRED=0 -) else ( - set CYGWIN_FLAGS= + if "%CYGWIN_INSTALL_PACKAGES%" neq "" ( + set CYGWIN_FLAGS=--upgrade-also + set CYGWIN_UPGRADE_REQUIRED=0 + ) ) if "%CYGWIN_INSTALL_PACKAGES%" neq "" "%CYG_ROOT%\setup-x86_64.exe" --quiet-mode --no-shortcuts --no-startmenu --no-desktop --only-site --root "%CYG_ROOT%" --site "%CYG_MIRROR%" --local-package-dir "%CYG_CACHE%" %CYGWIN_FLAGS% --packages %CYGWIN_INSTALL_PACKAGES:~1% for %%P in (%CYGWIN_COMMANDS%) do "%CYG_ROOT%\bin\%%P.exe" --version 2> nul > nul || set CYGWIN_UPGRADE_REQUIRED=1 From d49f2d67b8daad953be7d1458253c9f339e562cf Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Sun, 6 Oct 2024 21:19:25 +0100 Subject: [PATCH 7/8] Logic error in configure.ac --- configure | 18 +++++++++--------- configure.ac | 16 ++++++++-------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/configure b/configure index 69efedff733e..d8cd5948b397 100755 --- a/configure +++ b/configure @@ -16028,15 +16028,6 @@ then : esac fi -case $enable_native_toplevel,$natdynlink in #( - yes,false) : - as_fn_error $? "The native toplevel requires native dynlink support" "$LINENO" 5 ;; #( - yes,*) : - install_ocamlnat=true ;; #( - *) : - install_ocamlnat=false ;; -esac - # Try to work around the Skylake/Kaby Lake processor bug. case "$ocaml_cc_vendor,$host" in #( *gcc*,x86_64-*|*gcc*,i686-*) : @@ -16332,6 +16323,15 @@ else $as_nop natdynlink_archive="" fi +case $enable_native_toplevel,$natdynlink in #( + yes,false) : + as_fn_error $? "The native toplevel requires native dynlink support" "$LINENO" 5 ;; #( + yes,*) : + install_ocamlnat=true ;; #( + *) : + install_ocamlnat=false ;; +esac + printf "%s\n" "#define OCAML_OS_TYPE \"$ostype\"" >>confdefs.h diff --git a/configure.ac b/configure.ac index 5249b69d4d42..1c702ebc58d3 100644 --- a/configure.ac +++ b/configure.ac @@ -1400,14 +1400,6 @@ AS_IF([test x"$supports_shared_libraries" = 'xtrue'], [aarch64-*-netbsd*], [natdynlink=true], [riscv*-*-linux*], [natdynlink=true])]) -AS_CASE([$enable_native_toplevel,$natdynlink], - [yes,false], - [AC_MSG_ERROR(m4_normalize([ - The native toplevel requires native dynlink support]))], - [yes,*], - [install_ocamlnat=true], - [install_ocamlnat=false]) - # Try to work around the Skylake/Kaby Lake processor bug. AS_CASE(["$ocaml_cc_vendor,$host"], [*gcc*,x86_64-*|*gcc*,i686-*], @@ -1567,6 +1559,14 @@ AS_IF([$natdynlink], [natdynlink_archive="dynlink.cmxa"], [natdynlink_archive=""]) +AS_CASE([$enable_native_toplevel,$natdynlink], + [yes,false], + [AC_MSG_ERROR(m4_normalize([ + The native toplevel requires native dynlink support]))], + [yes,*], + [install_ocamlnat=true], + [install_ocamlnat=false]) + AC_DEFINE_UNQUOTED([OCAML_OS_TYPE], ["$ostype"]) AC_CHECK_TOOL([DIRECT_LD],[ld]) From 6ff1ea2379dfccac5cf7425156a47a170c67d57c Mon Sep 17 00:00:00 2001 From: David Allsopp Date: Thu, 17 Oct 2024 13:18:35 +0100 Subject: [PATCH 8/8] Fix inclusion of libraries when partial linking ld -r (certainly in GNU binutils) has an empty search path - co-opt the MSVC search code and always resolve libraries when partial linking, except this time _ignore_ the ones which are missing. This seems to fit the rest of -output-complete-obj, given that the _standard_ C libraries are also omitted (-lm, -lpthread, etc.) --- testsuite/tools/test_in_prefix.ml | 11 ++++------- utils/ccomp.ml | 30 +++++++++++++++++------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/testsuite/tools/test_in_prefix.ml b/testsuite/tools/test_in_prefix.ml index ae31ef769ef4..deefd7ddbf75 100644 --- a/testsuite/tools/test_in_prefix.ml +++ b/testsuite/tools/test_in_prefix.ml @@ -865,14 +865,11 @@ let compile_test ~original env bindir = | Output_complete_obj(C_ocamlc, Shared) -> true, false, ["-output-complete-obj"], true, None | Output_complete_obj(C_ocamlopt, Static) -> - (* At the moment, the partial linker will pass -lzstd to ld -r which - will (normally) fail). Until this is done, pass the libraries - manually. *) - false, true, ["-output-complete-obj"; "-noautolink"], true, - Some ocamlcommon_native_c_libraries + false, true, ["-output-complete-obj"], true, + Some Config.comprmarsh_c_libraries | Output_complete_obj(C_ocamlopt, Shared) -> - true, true, ["-output-complete-obj"; "-noautolink"], true, - Some ocamlcommon_native_c_libraries + true, true, ["-output-complete-obj"], true, + Some Config.comprmarsh_c_libraries | Output_complete_exe Static -> false, false, ["-output-complete-exe"], false, None | Output_complete_exe Shared -> diff --git a/utils/ccomp.ml b/utils/ccomp.ml index defe4d2a4b92..96dd7ade1c1a 100644 --- a/utils/ccomp.ml +++ b/utils/ccomp.ml @@ -147,16 +147,15 @@ let create_archive archive file_list = (quote_files ~response_files:Config.ar_supports_response_files file_list)) -let expand_libname cclibs = - cclibs |> List.map (fun cclib -> - if String.starts_with ~prefix:"-l" cclib then - let libname = - "lib" ^ String.sub cclib 2 (String.length cclib - 2) ^ Config.ext_lib in - try - Load_path.find libname - with Not_found -> - libname - else cclib) +let expand_libname cclib = + if String.starts_with ~prefix:"-l" cclib then + let libname = + "lib" ^ String.sub cclib 2 (String.length cclib - 2) ^ Config.ext_lib in + try + Some (Load_path.find libname) + with Not_found -> + None + else Some cclib type link_mode = | Exe @@ -176,11 +175,16 @@ let call_linker mode output_name files extra = Profile.record_call "c-linker" (fun () -> let cmd = if mode = Partial then - let (l_prefix, files) = + let l_prefix = match Config.ccomp_type with - | "msvc" -> ("/libpath:", expand_libname files) - | _ -> ("-L", files) + | "msvc" -> "/libpath:" + | _ -> "-L" in + (* For partial linking, only include -llib if -llib can be found in the + current search path. For ld -r, this PATH is (usually) limited to the + -L directories. This should cause OCaml libraries to be linked, but + not any system libraries mentioned in .cma/.cmxa files. *) + let files = List.filter_map expand_libname files in Printf.sprintf "%s%s %s %s %s" Config.native_pack_linker (Filename.quote output_name)