From 417ecb3d54eb3ea38b20004f08b7d442ea10d847 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Tue, 26 Nov 2024 06:36:22 -0300 Subject: [PATCH 1/8] Only add symbols that are explicit entrypoints --- src/precompile_utils.c | 6 ++---- src/staticdata.c | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 81c60ba70d29f..662b98d17ebf0 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -374,9 +374,8 @@ static void *jl_precompile_trimmed(size_t world) jl_value_t *ccallable = NULL; JL_GC_PUSH2(&m, &ccallable); jl_method_instance_t *mi; - while (1) - { - mi = (jl_method_instance_t*)arraylist_pop(jl_entrypoint_mis); + for (size_t i = 0; i < jl_entrypoint_mis->len ; i++) { + mi = (jl_method_instance_t*)jl_entrypoint_mis->items[i]; if (mi == NULL) break; assert(jl_is_method_instance(mi)); @@ -386,7 +385,6 @@ static void *jl_precompile_trimmed(size_t world) if (ccallable) jl_array_ptr_1d_push(m, ccallable); } - jl_cgparams_t params = jl_default_cgparams; params.trim = jl_options.trim; void *native_code = jl_create_native(m, NULL, ¶ms, 0, /* imaging */ 1, 0, diff --git a/src/staticdata.c b/src/staticdata.c index c2e8d7e3956ea..e31d498dae895 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1741,8 +1741,20 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED else { newm->nroots_sysimg = m->roots ? jl_array_len(m->roots) : 0; } - if (m->ccallable) - arraylist_push(&s->ccallable_list, (void*)reloc_offset); + if (m->ccallable) { + int should_push = !jl_options.trim; + for (size_t i = 0; !should_push && i < jl_entrypoint_mis->len ; i++) { + jl_method_instance_t* mi = (jl_method_instance_t*)jl_entrypoint_mis->items[i]; + assert(jl_is_method_instance(mi)); + jl_method_t *ccallable = mi->def.method; + if (ccallable == m) { + should_push = 1; + break; + } + } + if (should_push) + arraylist_push(&s->ccallable_list, (void*)reloc_offset); + } } else if (jl_is_method_instance(v)) { assert(f == s->s); From 69eee92cee9cd7bf89f26b5f7e3f4a63d85744bb Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Wed, 22 Jan 2025 15:40:30 -0300 Subject: [PATCH 2/8] Change ccallable handling for entrypoints --- src/gf.c | 2 +- src/init.c | 6 +++--- src/julia_internal.h | 2 +- src/precompile_utils.c | 15 ++++----------- src/staticdata.c | 8 +++----- 5 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/gf.c b/src/gf.c index 710dda208f0b2..c9d477dbf1747 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3378,7 +3378,7 @@ JL_DLLEXPORT int jl_add_entrypoint(jl_tupletype_t *types) return 0; JL_GC_PROMISE_ROOTED(mi); if (jl_generating_output() && jl_options.trim) { - arraylist_push(jl_entrypoint_mis, mi); + arraylist_push(jl_entrypoint_list, mi); } return 1; } diff --git a/src/init.c b/src/init.c index e69467c75bd73..9fb159c973f41 100644 --- a/src/init.c +++ b/src/init.c @@ -44,7 +44,7 @@ extern BOOL (WINAPI *hSymRefreshModuleList)(HANDLE); // list of modules being deserialized with __init__ methods jl_array_t *jl_module_init_order; -arraylist_t *jl_entrypoint_mis; +arraylist_t *jl_entrypoint_list; JL_DLLEXPORT size_t jl_page_size; @@ -931,8 +931,8 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ } if (jl_options.trim) { - jl_entrypoint_mis = (arraylist_t *)malloc_s(sizeof(arraylist_t)); - arraylist_new(jl_entrypoint_mis, 0); + jl_entrypoint_list = (arraylist_t *)malloc_s(sizeof(arraylist_t)); + arraylist_new(jl_entrypoint_list, 0); } if (jl_options.handle_signals == JL_OPTIONS_HANDLE_SIGNALS_ON) diff --git a/src/julia_internal.h b/src/julia_internal.h index 0da6d412c8a49..45332b058ee07 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -908,7 +908,7 @@ extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; extern jl_genericmemory_t *jl_global_roots_list JL_GLOBALLY_ROOTED; extern jl_genericmemory_t *jl_global_roots_keyset JL_GLOBALLY_ROOTED; -extern arraylist_t *jl_entrypoint_mis; +extern arraylist_t *jl_entrypoint_list; JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val, int insert) JL_GLOBALLY_ROOTED; extern jl_svec_t *precompile_field_replace JL_GLOBALLY_ROOTED; diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 56b0659453bf1..e0a471d393592 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -349,7 +349,7 @@ static int enq_ccallable_entrypoints_(jl_typemap_entry_t *def, void *closure) if (m->external_mt) return 1; if (m->ccallable) - jl_add_entrypoint((jl_tupletype_t*)jl_svecref(m->ccallable, 1)); + arraylist_push(jl_entrypoint_list, m->ccallable); return 1; } @@ -367,16 +367,9 @@ static void *jl_precompile_trimmed(size_t world) { // array of MethodInstances and ccallable aliases to include in the output jl_array_t *m = jl_alloc_vec_any(0); - jl_value_t *ccallable = NULL; - JL_GC_PUSH2(&m, &ccallable); - jl_method_instance_t *mi; - for (size_t i = 0; i < jl_entrypoint_mis->len ; i++) { - mi = (jl_method_instance_t*)jl_entrypoint_mis->items[i]; - assert(jl_is_method_instance(mi)); - jl_array_ptr_1d_push(m, (jl_value_t*)mi); - ccallable = (jl_value_t *)mi->def.method->ccallable; - if (ccallable) - jl_array_ptr_1d_push(m, ccallable); + JL_GC_PUSH(&m); + for (size_t i = 0; i < jl_entrypoint_list->len ; i++) { + jl_array_ptr_1d_push(m, (jl_value_t*)(jl_entrypoint_list->items[i])); } void *native_code = jl_create_native(m, NULL, jl_options.trim, 0, world); diff --git a/src/staticdata.c b/src/staticdata.c index 63baf3b147b0c..f7e1669b70fc5 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1790,11 +1790,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED } if (m->ccallable) { int should_push = !jl_options.trim; - for (size_t i = 0; !should_push && i < jl_entrypoint_mis->len ; i++) { - jl_method_instance_t* mi = (jl_method_instance_t*)jl_entrypoint_mis->items[i]; - assert(jl_is_method_instance(mi)); - jl_method_t *ccallable = mi->def.method; - if (ccallable == m) { + for (size_t i = 0; !should_push && i < jl_entrypoint_list->len ; i++) { + jl_value_t* val = jl_entrypoint_list->items[i]; + if ((jl_value_t *)m->ccallable == val) { should_push = 1; break; } From 49cc1ab18ae10a3b85d7c4d4e6c0cfafd4dc1b10 Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Wed, 22 Jan 2025 15:41:35 -0300 Subject: [PATCH 3/8] Fix typo --- src/precompile_utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/precompile_utils.c b/src/precompile_utils.c index e0a471d393592..f8ec6eced4241 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -367,7 +367,7 @@ static void *jl_precompile_trimmed(size_t world) { // array of MethodInstances and ccallable aliases to include in the output jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH(&m); + JL_GC_PUSH1(&m); for (size_t i = 0; i < jl_entrypoint_list->len ; i++) { jl_array_ptr_1d_push(m, (jl_value_t*)(jl_entrypoint_list->items[i])); } From 4f1c51cf2b0a67abd74d854f9e940fe509f5b66f Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 22 Jan 2025 14:46:48 -0500 Subject: [PATCH 4/8] Update src/staticdata.c Co-authored-by: Cody Tapscott <84105208+topolarity@users.noreply.github.com> --- src/staticdata.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index f7e1669b70fc5..de5466e39e76a 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1789,16 +1789,17 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED newm->nroots_sysimg = m->roots ? jl_array_len(m->roots) : 0; } if (m->ccallable) { - int should_push = !jl_options.trim; - for (size_t i = 0; !should_push && i < jl_entrypoint_list->len ; i++) { - jl_value_t* val = jl_entrypoint_list->items[i]; - if ((jl_value_t *)m->ccallable == val) { - should_push = 1; - break; + if (jl_options.trim == JL_TRIM_NO) { + arraylist_push(&s->ccallable_list, (void*)reloc_offset); + } else { + for (size_t i = 0; i < jl_entrypoint_list->len; i++) { + jl_value_t* val = (jl_value_t*)jl_entrypoint_list->items[i]; + if ((jl_value_t *)m->ccallable == val) { + arraylist_push(&s->ccallable_list, (void*)reloc_offset); + break; + } } } - if (should_push) - arraylist_push(&s->ccallable_list, (void*)reloc_offset); } } else if (jl_is_method_instance(v)) { From 36020d92d48c4ec63ca97198c8d92ee2459f699d Mon Sep 17 00:00:00 2001 From: gbaraldi Date: Wed, 22 Jan 2025 16:58:46 -0300 Subject: [PATCH 5/8] Add test for ccallable function called by ccallable function. --- test/trimming/hello.jl | 7 +++++++ test/trimming/trimming.jl | 1 + 2 files changed, 8 insertions(+) diff --git a/test/trimming/hello.jl b/test/trimming/hello.jl index 307bf820f325b..e2c521818a3bd 100644 --- a/test/trimming/hello.jl +++ b/test/trimming/hello.jl @@ -1,6 +1,13 @@ module MyApp + +Base.@ccallable function foo()::Cint + Cint(1) +end + Base.@ccallable function main()::Cint println(Core.stdout, "Hello, world!") + u = foo() + println(Core.stdout, u) return 0 end end diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl index dfacae7f8e531..5b47797dca0f8 100644 --- a/test/trimming/trimming.jl +++ b/test/trimming/trimming.jl @@ -3,5 +3,6 @@ using Test exe_path = joinpath(@__DIR__, "hello"*splitext(Base.julia_exename())[2]) @test readchomp(`$exe_path`) == "Hello, world!" +@test readchomp(`$exe_path`) == "1" @test filesize(exe_path) < filesize(unsafe_string(Base.JLOptions().image_file))/10 From fd6c780a2a5c09ba1581849186ab5b79ec7d1a70 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 23 Jan 2025 19:36:50 -0500 Subject: [PATCH 6/8] fix for ccallables with trimming --- src/precompile_utils.c | 13 ++++++++++--- src/staticdata.c | 4 ++-- test/trimming/Makefile | 2 +- test/trimming/trimming.jl | 3 +-- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/precompile_utils.c b/src/precompile_utils.c index f8ec6eced4241..44b6e093b7460 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -349,7 +349,7 @@ static int enq_ccallable_entrypoints_(jl_typemap_entry_t *def, void *closure) if (m->external_mt) return 1; if (m->ccallable) - arraylist_push(jl_entrypoint_list, m->ccallable); + jl_add_entrypoint((jl_tupletype_t*)jl_svecref(m->ccallable, 1)); return 1; } @@ -367,9 +367,16 @@ static void *jl_precompile_trimmed(size_t world) { // array of MethodInstances and ccallable aliases to include in the output jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH1(&m); + jl_value_t *ccallable = NULL; + JL_GC_PUSH2(&m, &ccallable); + jl_method_instance_t *mi; for (size_t i = 0; i < jl_entrypoint_list->len ; i++) { - jl_array_ptr_1d_push(m, (jl_value_t*)(jl_entrypoint_list->items[i])); + mi = (jl_method_instance_t*)jl_entrypoint_list->items[i]; + assert(jl_is_method_instance(mi)); + jl_array_ptr_1d_push(m, (jl_value_t*)mi); + ccallable = (jl_value_t *)mi->def.method->ccallable; + if (ccallable) + jl_array_ptr_1d_push(m, ccallable); } void *native_code = jl_create_native(m, NULL, jl_options.trim, 0, world); diff --git a/src/staticdata.c b/src/staticdata.c index de5466e39e76a..7e05d734505cd 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1793,8 +1793,8 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED arraylist_push(&s->ccallable_list, (void*)reloc_offset); } else { for (size_t i = 0; i < jl_entrypoint_list->len; i++) { - jl_value_t* val = (jl_value_t*)jl_entrypoint_list->items[i]; - if ((jl_value_t *)m->ccallable == val) { + jl_value_t *val = (jl_value_t*)jl_entrypoint_list->items[i]; + if (m == ((jl_method_instance_t*)val)->def.method) { arraylist_push(&s->ccallable_list, (void*)reloc_offset); break; } diff --git a/test/trimming/Makefile b/test/trimming/Makefile index d2da21eb71a88..4a889726204b8 100644 --- a/test/trimming/Makefile +++ b/test/trimming/Makefile @@ -33,7 +33,7 @@ LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) -ljulia-internal release: hello$(EXE) hello.o: $(SRCDIR)/hello.jl $(BUILDSCRIPT) - $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.so --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $(SRCDIR)/hello.jl --output-exe true + $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.so --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $(SRCDIR)/hello.jl --output-exe false init.o: $(SRCDIR)/init.c $(CC) -c -o $@ $< $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl index 5b47797dca0f8..fb4d5f95326f5 100644 --- a/test/trimming/trimming.jl +++ b/test/trimming/trimming.jl @@ -2,7 +2,6 @@ using Test exe_path = joinpath(@__DIR__, "hello"*splitext(Base.julia_exename())[2]) -@test readchomp(`$exe_path`) == "Hello, world!" -@test readchomp(`$exe_path`) == "1" +@test readchomp(`$exe_path`) == "Hello, world!\n1" @test filesize(exe_path) < filesize(unsafe_string(Base.JLOptions().image_file))/10 From 28125555d60d7fac33bf9011cfef9cd827ad9e5c Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 24 Jan 2025 15:59:59 -0500 Subject: [PATCH 7/8] include CodeInstances for ccallables with const_api in inference output This is only needed because `jl_reinit_ccallable` looks at method metadata that trimming will normally remove for const_api code. --- Compiler/src/typeinfer.jl | 6 ++++-- test/trimming/hello.jl | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index e3896870d82b8..f239300de82f2 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1277,7 +1277,9 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: # then we want to compile and emit this if item.def.primary_world <= this_world <= item.def.deleted_world ci = typeinf_ext(interp, item, SOURCE_MODE_NOT_REQUIRED) - ci isa CodeInstance && !use_const_api(ci) && push!(tocompile, ci) + if ci isa CodeInstance && (!use_const_api(ci) || isdefined(ci.def.def, :ccallable)) + push!(tocompile, ci) + end end elseif item isa SimpleVector push!(codeinfos, item[1]::Type) @@ -1292,7 +1294,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: mi = get_ci_mi(callee) def = mi.def if use_const_api(callee) - src = codeinfo_for_const(interp, mi, code.rettype_const) + src = codeinfo_for_const(interp, mi, callee.rettype_const) elseif haskey(interp.codegen, callee) src = interp.codegen[callee] elseif isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 && !trim diff --git a/test/trimming/hello.jl b/test/trimming/hello.jl index e2c521818a3bd..33d60aab343fa 100644 --- a/test/trimming/hello.jl +++ b/test/trimming/hello.jl @@ -1,9 +1,17 @@ module MyApp +# a ccallable that is used but not declared as an entrypoint. should not have a C name generated. Base.@ccallable function foo()::Cint Cint(1) end +# a ccallable that is an entrypoint but uses const_api. +Base.@ccallable function foo2()::Cint + Cint(2) +end + +Base.Experimental.entrypoint(foo2, ()) + Base.@ccallable function main()::Cint println(Core.stdout, "Hello, world!") u = foo() From 46d6b24cf0163a8ac6456f35ca3dff3a2de3c25d Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Fri, 24 Jan 2025 15:59:59 -0500 Subject: [PATCH 8/8] include CodeInstances with const_api in inference output when trimming --- Compiler/src/typeinfer.jl | 6 ++++-- test/trimming/hello.jl | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index e3896870d82b8..7ff691c0360fa 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1277,7 +1277,9 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: # then we want to compile and emit this if item.def.primary_world <= this_world <= item.def.deleted_world ci = typeinf_ext(interp, item, SOURCE_MODE_NOT_REQUIRED) - ci isa CodeInstance && !use_const_api(ci) && push!(tocompile, ci) + if ci isa CodeInstance && (!use_const_api(ci) || trim) + push!(tocompile, ci) + end end elseif item isa SimpleVector push!(codeinfos, item[1]::Type) @@ -1292,7 +1294,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: mi = get_ci_mi(callee) def = mi.def if use_const_api(callee) - src = codeinfo_for_const(interp, mi, code.rettype_const) + src = codeinfo_for_const(interp, mi, callee.rettype_const) elseif haskey(interp.codegen, callee) src = interp.codegen[callee] elseif isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 && !trim diff --git a/test/trimming/hello.jl b/test/trimming/hello.jl index e2c521818a3bd..33d60aab343fa 100644 --- a/test/trimming/hello.jl +++ b/test/trimming/hello.jl @@ -1,9 +1,17 @@ module MyApp +# a ccallable that is used but not declared as an entrypoint. should not have a C name generated. Base.@ccallable function foo()::Cint Cint(1) end +# a ccallable that is an entrypoint but uses const_api. +Base.@ccallable function foo2()::Cint + Cint(2) +end + +Base.Experimental.entrypoint(foo2, ()) + Base.@ccallable function main()::Cint println(Core.stdout, "Hello, world!") u = foo()