Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dynamic import #5703

Merged
merged 14 commits into from
Apr 23, 2023
Merged
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions jscomp/core/js_packages_info.ml
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ let is_runtime_package (x : t) = x.name = Pkg_runtime

let iter (x : t) cb = Ext_list.iter x.module_systems cb

let map (x : t) cb = Ext_list.map x.module_systems cb

(* let equal (x : t) ({name; module_systems}) =
x.name = name &&
Ext_list.for_all2_no_exn
Expand Down
2 changes: 2 additions & 0 deletions jscomp/core/js_packages_info.mli
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ val same_package_by_name : t -> t -> bool

val iter : t -> (package_info -> unit) -> unit

val map : t -> (package_info -> 'a) -> 'a list

val empty : t

val from_name : string -> t
Expand Down
2 changes: 1 addition & 1 deletion jscomp/core/lam_analysis.ml
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ let rec no_side_effects (lam : Lam.t) : bool =
| Pcreate_extension _ | Pjs_typeof | Pis_null | Pis_not_none | Psome
| Psome_not_nest | Pis_undefined | Pis_null_undefined | Pnull_to_opt
| Pundefined_to_opt | Pnull_undefined_to_opt | Pjs_fn_make _ | Pjs_fn_make_unit
| Pjs_object_create _
| Pjs_object_create _ | Pimport
(* TODO: check *)
| Pbytes_to_string | Pmakeblock _
(* whether it's mutable or not *)
Expand Down
270 changes: 138 additions & 132 deletions jscomp/core/lam_compile.ml

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions jscomp/core/lam_compile.mli
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,6 @@
(** Compile single lambda IR to JS IR *)

val compile_recursive_lets :
Lam_compile_context.t -> (Ident.t * Lam.t) list -> Js_output.t
output_prefix:string -> Lam_compile_context.t -> (Ident.t * Lam.t) list -> Js_output.t

val compile_lambda : Lam_compile_context.t -> Lam.t -> Js_output.t
val compile_lambda : output_prefix:string -> Lam_compile_context.t -> Lam.t -> Js_output.t
16 changes: 8 additions & 8 deletions jscomp/core/lam_compile_main.ml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
(* module S = Js_stmt_make *)


let compile_group (meta : Lam_stats.t)
let compile_group output_prefix (meta : Lam_stats.t)
(x : Lam_group.t) : Js_output.t =
match x with
(*
Expand All @@ -60,20 +60,20 @@ let compile_group (meta : Lam_stats.t)
(* let lam = Optimizer.simplify_lets [] lam in *)
(* can not apply again, it's wrong USE it with care*)
(* ([Js_stmt_make.comment (Gen_of_env.query_type id env )], None) ++ *)
Lam_compile.compile_lambda { continuation = Declare (kind, id);
Lam_compile.compile_lambda ~output_prefix { continuation = Declare (kind, id);
jmp_table = Lam_compile_context.empty_handler_map;
meta
} lam

| Recursive id_lams ->
Lam_compile.compile_recursive_lets
Lam_compile.compile_recursive_lets ~output_prefix
{ continuation = EffectCall Not_tail;
jmp_table = Lam_compile_context.empty_handler_map;
meta
}
id_lams
| Nop lam -> (* TODO: Side effect callls, log and see statistics *)
Lam_compile.compile_lambda {continuation = EffectCall Not_tail;
Lam_compile.compile_lambda ~output_prefix {continuation = EffectCall Not_tail;
jmp_table = Lam_compile_context.empty_handler_map;
meta
} lam
Expand Down Expand Up @@ -122,7 +122,7 @@ let _j = Js_pass_debug.dump
it's used or not
*)
let compile
(output_prefix : string)
(output_prefix : string)
export_idents
(lam : Lambda.lambda) =
let export_ident_sets = Set_ident.of_list export_idents in
Expand Down Expand Up @@ -222,7 +222,7 @@ let maybe_pure = no_side_effects groups in
let () = Ext_log.dwarn ~__POS__ "\n@[[TIME:]Pre-compile: %f@]@." (Sys.time () *. 1000.) in
#endif
let body =
Ext_list.map groups (fun group -> compile_group meta group)
Ext_list.map groups (fun group -> compile_group output_prefix meta group)
|> Js_output.concat
|> Js_output.output_as_block
in
Expand Down Expand Up @@ -292,13 +292,13 @@ let lambda_as_module
: unit =
let package_info = Js_packages_state.get_packages_info () in
if Js_packages_info.is_empty package_info && !Js_config.js_stdout then begin
Js_dump_program.dump_deps_program ~output_prefix NodeJS lambda_output stdout
Js_dump_program.dump_deps_program ~output_prefix NodeJS (lambda_output) stdout
end else
Js_packages_info.iter package_info (fun {module_system; path; suffix} ->
let output_chan chan =
Js_dump_program.dump_deps_program ~output_prefix
module_system
lambda_output
(lambda_output)
chan in
let basename =
Ext_namespace.change_ext_ns_suffix
Expand Down
65 changes: 63 additions & 2 deletions jscomp/core/lam_compile_primitive.ml
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,44 @@ let ensure_value_unit (st : Lam_compile_context.continuation) e : E.t =
| EffectCall Not_tail -> e
(* NeedValue should return a meaningful expression*)

let translate loc (cxt : Lam_compile_context.t) (prim : Lam_primitive.t)
(args : J.expression list) : J.expression =
let module_of_expression = function
| J.Var (J.Qualified (module_id, value)) -> [ (module_id, value) ]
| _ -> []

let get_module_system () =
let package_info = Js_packages_state.get_packages_info () in
let module_system =
if Js_packages_info.is_empty package_info && !Js_config.js_stdout then
[Js_packages_info.NodeJS]
else Js_packages_info.map package_info (fun {module_system} -> module_system)
in
match module_system with
| [module_system] -> module_system
| _ -> NodeJS
Comment on lines +43 to +52
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/rescript-lang/rescript-compiler/blob/master/jscomp/core/lam_compile_main.ml#L294-L297

Not sure in what cases the module_system in Js_packages_info can be multiple in one runtime. For now, I've handled the case where there is only one.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps we need some tests on fairly large projects.


let import_of_path path =
E.call
~info:{ arity = Full; call_info = Call_na }
(E.js_global "import")
[ E.str path ]

let wrap_then import value =
let arg = Ident.create "m" in
E.call
~info:{ arity = Full; call_info = Call_na }
(E.dot import "then")
[
E.ocaml_fun ~return_unit:false ~async:false ~oneUnitArg:false [ arg ]
[
{
statement_desc = J.Return (E.dot (E.var arg) value);
comment = None;
};
];
]

let translate output_prefix loc (cxt : Lam_compile_context.t)
(prim : Lam_primitive.t) (args : J.expression list) : J.expression =
match prim with
| Pis_not_none -> Js_of_lam_option.is_not_none (Ext_list.singleton_exn args)
| Pcreate_extension s -> E.make_exception s
Expand Down Expand Up @@ -78,6 +114,31 @@ let translate loc (cxt : Lam_compile_context.t) (prim : Lam_primitive.t)
| _ -> E.runtime_call Js_runtime_modules.option "nullable_to_opt" args
)
| _ -> assert false)
(* Compile #import: The module argument for dynamic import is represented as a path,
and the module value is expressed through wrapping it with promise.then *)
| Pimport -> (
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add comment describing what the transformation does.

match args with
| [ e ] -> (
let output_dir = Filename.dirname output_prefix in

let module_id, module_value =
match module_of_expression e.expression_desc with
| [ module_ ] -> module_
| _ -> Location.raise_errorf ~loc
"Invalid argument: Dynamic import requires a module or module value that is a file as argument. Passing a value or local module is not allowed."
in

let path =
let module_system = get_module_system () in
Js_name_of_module_id.string_of_module_id module_id ~output_dir module_system
in

match module_value with
| Some value -> wrap_then (import_of_path path) value
| None -> import_of_path path)
| [] | _ ->
Location.raise_errorf ~loc
"Invalid argument: Dynamic import must take a single module or module value as its argument.")
| Pjs_function_length -> E.function_length (Ext_list.singleton_exn args)
| Pcaml_obj_length -> E.obj_length (Ext_list.singleton_exn args)
| Pis_null -> E.is_null (Ext_list.singleton_exn args)
Expand Down
1 change: 1 addition & 0 deletions jscomp/core/lam_compile_primitive.mli
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
*)

val translate :
string ->
Location.t ->
Lam_compile_context.t ->
Lam_primitive.t ->
Expand Down
1 change: 1 addition & 0 deletions jscomp/core/lam_convert.ml
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,7 @@ let convert (exports : Set_ident.t) (lam : Lambda.lambda) :
| "#nullable_to_opt" -> Pnull_undefined_to_opt
| "#null_to_opt" -> Pnull_to_opt
| "#is_nullable" -> Pis_null_undefined
| "#import" ->Pimport
| "#string_append" -> Pstringadd
| "#wrap_exn" -> Pwrap_exn
| "#obj_length" -> Pcaml_obj_length
Expand Down
2 changes: 2 additions & 0 deletions jscomp/core/lam_primitive.ml
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ type t =
| Pis_null
| Pis_undefined
| Pis_null_undefined
| Pimport
| Pjs_typeof
| Pjs_function_length
| Pcaml_obj_length
Expand Down Expand Up @@ -219,6 +220,7 @@ let eq_primitive_approx (lhs : t) (rhs : t) =
| Psome_not_nest -> rhs = Psome_not_nest
| Pis_undefined -> rhs = Pis_undefined
| Pis_null_undefined -> rhs = Pis_null_undefined
| Pimport -> rhs = Pimport
| Pjs_typeof -> rhs = Pjs_typeof
| Pisint -> rhs = Pisint
| Pis_poly_var_block -> rhs = Pis_poly_var_block
Expand Down
1 change: 1 addition & 0 deletions jscomp/core/lam_primitive.mli
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ type t =
| Pis_null
| Pis_undefined
| Pis_null_undefined
| Pimport
| Pjs_typeof
| Pjs_function_length
| Pcaml_obj_length
Expand Down
1 change: 1 addition & 0 deletions jscomp/core/lam_print.ml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ let primitive ppf (prim : Lam_primitive.t) =
| Pval_from_option_not_nest -> fprintf ppf "[?unbox-not-nest]"
| Pis_undefined -> fprintf ppf "[?undefined]"
| Pis_null_undefined -> fprintf ppf "[?null?undefined]"
| Pimport -> fprintf ppf "[import]"
| Pmakeblock (tag, _, Immutable) -> fprintf ppf "makeblock %i" tag
| Pmakeblock (tag, _, Mutable) -> fprintf ppf "makemutable %i" tag
| Pfield (n, field_info) -> (
Expand Down
33 changes: 33 additions & 0 deletions jscomp/frontend/ast_await.ml
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,36 @@ let create_await_expression (e : Parsetree.expression) =
{txt = Ldot (Ldot (Lident "Js", "Promise"), "unsafe_await"); loc}
in
Ast_helper.Exp.apply ~loc unsafe_await [(Nolabel, e)]

(* Transform `@res.await M` to unpack(@res.await Js.import(module(M: __M0__))) *)
let create_await_module_expression ~module_type_name (e : Parsetree.module_expr)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a comment explaining what this transformation does.

=
let open Ast_helper in
let remove_await_attribute =
List.filter (fun ((loc, _) : Parsetree.attribute) -> loc.txt != "res.await")
in
{
e with
pmod_desc =
Pmod_unpack
(create_await_expression
(Exp.apply ~loc:e.pmod_loc
(Exp.ident ~loc:e.pmod_loc
{
txt = Longident.Ldot (Lident "Js", "import");
loc = e.pmod_loc;
})
[
( Nolabel,
Exp.constraint_ ~loc:e.pmod_loc
(Exp.pack ~loc:e.pmod_loc
{
e with
pmod_attributes =
remove_await_attribute e.pmod_attributes;
})
(Typ.package ~loc:e.pmod_loc
{txt = Lident module_type_name; loc = e.pmod_loc}
[]) );
]));
}
66 changes: 61 additions & 5 deletions jscomp/frontend/bs_builtin_ppx.ml
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,13 @@ let local_module_name =
incr v;
"local_" ^ string_of_int !v

(* Unpack requires core_type package for type inference:
Generate a module type name eg. __Belt_List__*)
let local_module_type_name txt =
"_"
^ (Longident.flatten txt |> List.fold_left (fun ll l -> ll ^ "_" ^ l) "")
^ "__"

let expand_reverse (stru : Ast_structure.t) (acc : Ast_structure.t) :
Ast_structure.t =
if stru = [] then acc
Expand Down Expand Up @@ -456,14 +463,15 @@ let expand_reverse (stru : Ast_structure.t) (acc : Ast_structure.t) :
}
:: acc)

let rec structure_mapper (self : mapper) (stru : Ast_structure.t) =
let rec structure_mapper ~await_context (self : mapper) (stru : Ast_structure.t)
=
match stru with
| [] -> []
| item :: rest -> (
match item.pstr_desc with
| Pstr_extension (({txt = "bs.raw" | "raw"; loc}, payload), _attrs) ->
Ast_exp_handle_external.handle_raw_structure loc payload
:: structure_mapper self rest
:: structure_mapper ~await_context self rest
(* | Pstr_extension (({txt = "i"}, _),_)
->
structure_mapper self rest *)
Expand All @@ -483,10 +491,58 @@ let rec structure_mapper (self : mapper) (stru : Ast_structure.t) =
next
| PSig _ | PTyp _ | PPat _ ->
Location.raise_errorf ~loc "private extension is not support")
| _ -> expand_reverse acc (structure_mapper self rest)
| _ -> expand_reverse acc (structure_mapper ~await_context self rest)
in
aux [] stru
| _ -> self.structure_item self item :: structure_mapper self rest)
(* Dynamic import of module transformation: module M = @res.await Belt.List *)
| Pstr_module
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a comment explaining what this transformation does.

({pmb_expr = {pmod_desc = Pmod_ident {txt; loc}; pmod_attributes} as me}
as mb)
when Res_parsetree_viewer.hasAwaitAttribute pmod_attributes ->
let item = self.structure_item self item in
let safe_module_type_name = local_module_type_name txt in
let has_local_module_name =
Hashtbl.find_opt !await_context safe_module_type_name
in
(* module __Belt_List__ = module type of Belt.List *)
let module_type_decl =
match has_local_module_name with
| Some _ -> []
| None ->
let open Ast_helper in
Hashtbl.add !await_context safe_module_type_name safe_module_type_name;
[
Str.modtype ~loc
(Mtd.mk ~loc
{txt = safe_module_type_name; loc}
~typ:(Mty.typeof_ ~loc me));
]
in
module_type_decl
@ (* module M = @res.await Belt.List *)
{
item with
pstr_desc =
Pstr_module
{
mb with
pmb_expr =
Ast_await.create_await_module_expression
~module_type_name:safe_module_type_name mb.pmb_expr;
};
}
:: structure_mapper ~await_context self rest
| _ ->
self.structure_item self item :: structure_mapper ~await_context self rest
)

let structure_mapper ~await_context (self : mapper) (stru : Ast_structure.t) =
let await_saved = !await_context in
let result =
structure_mapper ~await_context:(ref (Hashtbl.create 10)) self stru
in
await_context := await_saved;
result

let mapper : mapper =
{
Expand All @@ -497,7 +553,7 @@ let mapper : mapper =
signature_item = signature_item_mapper;
value_bindings = Ast_tuple_pattern_flatten.value_bindings_mapper;
structure_item = structure_item_mapper;
structure = structure_mapper;
structure = structure_mapper ~await_context:(ref (Hashtbl.create 10));
(* Ad-hoc way to internalize stuff *)
label_declaration =
(fun self lbl ->
Expand Down
1 change: 1 addition & 0 deletions jscomp/others/js.ml
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ external toOption : 'a nullable -> 'a option = "#nullable_to_opt"
external undefinedToOption : 'a undefined -> 'a option = "#undefined_to_opt"
external nullToOption : 'a null -> 'a option = "#null_to_opt"
external isNullable : 'a nullable -> bool = "#is_nullable"
external import : 'a -> 'a promise = "#import"

external testAny : 'a -> bool = "#is_nullable"
(** The same as {!test} except that it is more permissive on the types of input *)
Expand Down
2 changes: 2 additions & 0 deletions jscomp/runtime/js.ml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ external nullToOption : 'a null -> 'a option = "#null_to_opt"

external isNullable : 'a nullable -> bool = "#is_nullable"

external import : 'a -> 'a promise = "#import"

(** The same as {!test} except that it is more permissive on the types of input *)
external testAny : 'a -> bool = "#is_nullable"

Expand Down
Loading