Merge pull request #6808 from roc-lang/process-host

Update preprocess host API

crates/cli/src/lib.rs

@@ -79,6 +79,9 @@ pub const GLUE_DIR: &str = "GLUE_DIR";
 pub const GLUE_SPEC: &str = "GLUE_SPEC";
 pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES";
 pub const ARGS_FOR_APP: &str = "ARGS_FOR_APP";
+pub const FLAG_PP_HOST: &str = "host";
+pub const FLAG_PP_PLATFORM: &str = "platform";
+pub const FLAG_PP_DYLIB: &str = "lib";
 
 const VERSION: &str = include_str!("../../../version.txt");
 const DEFAULT_GENERATED_DOCS_DIR: &str = "generated-docs";
@@ -400,18 +403,29 @@ pub fn build_app() -> Command {
         .subcommand(Command::new(CMD_PREPROCESS_HOST)
             .about("Runs the surgical linker preprocessor to generate `.rh` and `.rm` files.")
             .arg(
-                Arg::new(ROC_FILE)
-                    .help("The .roc file for an app using the platform")
+                Arg::new(FLAG_PP_HOST)
+                    .help("Path to the host executable where the app was linked dynamically")
                     .value_parser(value_parser!(PathBuf))
                     .required(true)
             )
             .arg(
-                Arg::new(FLAG_TARGET)
-                    .long(FLAG_TARGET)
-                    .help("Choose a different target")
-                    .default_value(Into::<&'static str>::into(Target::default()))
-                    .value_parser(build_target_values_parser)
-                    .required(false),
+                Arg::new(FLAG_PP_PLATFORM)
+                    .help("Path to the platform/main.roc file")
+                    .value_parser(value_parser!(PathBuf))
+                    .required(true)
+            )
+            .arg(
+                Arg::new(FLAG_PP_DYLIB)
+                    .help("Path to a stubbed app dynamic library (e.g. roc build --lib app.roc)")
+                    .value_parser(value_parser!(PathBuf))
+                    .required(true)
+            )
+            .arg(
+                Arg::new(FLAG_VERBOSE)
+                    .long(FLAG_VERBOSE)
+                    .help("Print detailed information while pre-processing host")
+                    .action(ArgAction::SetTrue)
+                    .required(false)
             )
         )
         .arg(flag_optimize)

crates/cli/src/main.rs

@@ -6,8 +6,8 @@ use roc_cli::{
     build_app, format_files, format_src, test, BuildConfig, FormatMode, CMD_BUILD, CMD_CHECK,
     CMD_DEV, CMD_DOCS, CMD_FORMAT, CMD_GEN_STUB_LIB, CMD_GLUE, CMD_PREPROCESS_HOST, CMD_REPL,
     CMD_RUN, CMD_TEST, CMD_VERSION, DIRECTORY_OR_FILES, FLAG_CHECK, FLAG_DEV, FLAG_LIB, FLAG_MAIN,
-    FLAG_NO_LINK, FLAG_OUTPUT, FLAG_STDIN, FLAG_STDOUT, FLAG_TARGET, FLAG_TIME, GLUE_DIR,
-    GLUE_SPEC, ROC_FILE,
+    FLAG_NO_LINK, FLAG_OUTPUT, FLAG_PP_DYLIB, FLAG_PP_HOST, FLAG_PP_PLATFORM, FLAG_STDIN,
+    FLAG_STDOUT, FLAG_TARGET, FLAG_TIME, GLUE_DIR, GLUE_SPEC, ROC_FILE,
 };
 use roc_docs::generate_docs_html;
 use roc_error_macros::user_error;
@@ -136,31 +136,48 @@ fn main() -> io::Result<()> {
             Ok(0)
         }
         Some((CMD_PREPROCESS_HOST, matches)) => {
-            let input_path = matches.get_one::<PathBuf>(ROC_FILE).unwrap();
+            let preprocess_host_err =
+                { |msg: String| user_error!("\n\n ERROR PRE-PROCESSING HOST: {}\n\n", msg) };
+
+            let host_path = matches.get_one::<PathBuf>(FLAG_PP_HOST).unwrap();
+            if !host_path.is_file() {
+                preprocess_host_err(format!(
+                    "Expected to find the host executable file at {}",
+                    &host_path.display()
+                ));
+            }
+
+            let platform_path = matches.get_one::<PathBuf>(FLAG_PP_PLATFORM).unwrap();
+            if !platform_path.is_file() {
+                preprocess_host_err(format!(
+                    "Expected to find the platform/main.roc file at {}",
+                    &platform_path.display()
+                ));
+            }
+
+            let dylib_path = matches.get_one::<PathBuf>(FLAG_PP_DYLIB).unwrap();
+            if !dylib_path.is_file() {
+                preprocess_host_err(format!(
+                    "Expected to find the app stub dynamic library file at {}",
+                    dylib_path.display()
+                ));
+            }
             let target = matches
                 .get_one::<String>(FLAG_TARGET)
                 .and_then(|s| Target::from_str(s).ok())
                 .unwrap_or_default();
 
-            let function_kind = FunctionKind::LambdaSet;
-            let (platform_path, stub_lib, stub_dll_symbols) = roc_linker::generate_stub_lib(
-                input_path,
-                RocCacheDir::Persistent(cache::roc_cache_dir().as_path()),
-                target,
-                function_kind,
-            );
+            let verbose_and_time = matches.get_one::<bool>(roc_cli::FLAG_VERBOSE).unwrap();
 
-            // TODO: pipeline the executable location through here.
-            // Currently it is essentally hardcoded as platform_path/dynhost.
             roc_linker::preprocess_host(
                 target,
-                &platform_path.with_file_name("main.roc"),
-                // The target triple string must be derived from the triple to convert from the generic
-                // `system` target to the exact specific target.
-                &platform_path.with_file_name(format!("{}.rh", target)),
-                &stub_lib,
-                &stub_dll_symbols,
+                host_path,
+                platform_path,
+                dylib_path,
+                *verbose_and_time,
+                *verbose_and_time,
             );
+
             Ok(0)
         }
         Some((CMD_BUILD, matches)) => {

crates/compiler/build/src/program.rs

@@ -1169,22 +1169,23 @@ fn build_and_preprocess_host_lowlevel(
     opt_level: OptLevel,
     target: Target,
     platform_main_roc: &Path,
-    preprocessed_host_path: &Path,
+    _preprocessed_host_path: &Path,
     stub_dll_symbols: &[String],
 ) {
     let stub_lib =
         roc_linker::generate_stub_lib_from_loaded(target, platform_main_roc, stub_dll_symbols);
 
     debug_assert!(stub_lib.exists());
 
-    rebuild_host(opt_level, target, platform_main_roc, Some(&stub_lib));
+    let host_dest = rebuild_host(opt_level, target, platform_main_roc, Some(&stub_lib));
 
     roc_linker::preprocess_host(
         target,
+        host_dest.as_path(),
         platform_main_roc,
-        preprocessed_host_path,
         &stub_lib,
-        stub_dll_symbols,
+        false,
+        false,
     )
 }
 

crates/linker/src/elf.rs

@@ -15,9 +15,10 @@ use std::{
     io::{BufReader, BufWriter},
     mem,
     path::Path,
-    time::{Duration, Instant},
+    time::Instant,
 };
 
+use crate::util::{is_roc_definition, is_roc_undefined, report_timing};
 use crate::{
     align_by_constraint, align_to_offset_by_constraint, load_struct_inplace,
     load_struct_inplace_mut, load_structs_inplace_mut, open_mmap, open_mmap_mut,
@@ -103,26 +104,6 @@ impl Metadata {
     }
 }
 
-fn report_timing(label: &str, duration: Duration) {
-    println!("\t{:9.3} ms   {}", duration.as_secs_f64() * 1000.0, label,);
-}
-
-fn is_roc_symbol(sym: &object::Symbol) -> bool {
-    if let Ok(name) = sym.name() {
-        name.trim_start_matches('_').starts_with("roc_")
-    } else {
-        false
-    }
-}
-
-fn is_roc_definition(sym: &object::Symbol) -> bool {
-    sym.is_definition() && is_roc_symbol(sym)
-}
-
-fn is_roc_undefined(sym: &object::Symbol) -> bool {
-    sym.is_undefined() && is_roc_symbol(sym)
-}
-
 fn collect_roc_definitions<'a>(object: &object::File<'a, &'a [u8]>) -> MutMap<String, u64> {
     let mut vaddresses = MutMap::default();
 

crates/linker/src/lib.rs

@@ -19,6 +19,7 @@ use std::path::{Path, PathBuf};
 mod elf;
 mod macho;
 mod pe;
+mod util;
 
 mod generate_dylib;
 
@@ -365,27 +366,23 @@ fn stub_lib_is_up_to_date(target: Target, stub_lib_path: &Path, custom_names: &[
 
 pub fn preprocess_host(
     target: Target,
-    platform_main_roc: &Path,
-    preprocessed_path: &Path,
-    shared_lib: &Path,
-    stub_dll_symbols: &[String],
+    host_path: &Path,
+    platform_path: &Path,
+    dylib_path: &Path,
+    verbose: bool,
+    time: bool,
 ) {
-    let metadata_path = platform_main_roc.with_file_name(metadata_file_name(target));
-    let host_exe_path = if target.operating_system() == OperatingSystem::Windows {
-        platform_main_roc.with_file_name("dynhost.exe")
-    } else {
-        platform_main_roc.with_file_name("dynhost")
-    };
+    let preprocessed_path = platform_path.with_file_name(format!("{}.rh", target));
+    let metadata_path = platform_path.with_file_name(metadata_file_name(target));
 
     preprocess(
         target,
-        &host_exe_path,
+        host_path,
         &metadata_path,
-        preprocessed_path,
-        shared_lib,
-        stub_dll_symbols,
-        false,
-        false,
+        preprocessed_path.as_path(),
+        dylib_path,
+        verbose,
+        time,
     )
 }
 
@@ -397,7 +394,6 @@ fn preprocess(
     metadata_path: &Path,
     preprocessed_path: &Path,
     shared_lib: &Path,
-    stub_dll_symbols: &[String],
     verbose: bool,
     time: bool,
 ) {
@@ -433,7 +429,7 @@ fn preprocess(
                 host_exe_path,
                 metadata_path,
                 preprocessed_path,
-                stub_dll_symbols,
+                shared_lib,
                 verbose,
                 time,
             )

crates/linker/src/macho.rs

@@ -15,9 +15,10 @@ use std::{
     io::{BufReader, BufWriter},
     mem,
     path::Path,
-    time::{Duration, Instant},
+    time::Instant,
 };
 
+use crate::util::{is_roc_definition, is_roc_undefined, report_timing};
 use crate::{
     align_by_constraint, align_to_offset_by_constraint, load_struct_inplace,
     load_struct_inplace_mut, load_structs_inplace, load_structs_inplace_mut, open_mmap,
@@ -104,26 +105,6 @@ impl Metadata {
     }
 }
 
-fn report_timing(label: &str, duration: Duration) {
-    println!("\t{:9.3} ms   {}", duration.as_secs_f64() * 1000.0, label,);
-}
-
-fn is_roc_symbol(sym: &object::Symbol) -> bool {
-    if let Ok(name) = sym.name() {
-        name.trim_start_matches('_').starts_with("roc_")
-    } else {
-        false
-    }
-}
-
-fn is_roc_definition(sym: &object::Symbol) -> bool {
-    sym.is_definition() && is_roc_symbol(sym)
-}
-
-fn is_roc_undefined(sym: &object::Symbol) -> bool {
-    sym.is_undefined() && is_roc_symbol(sym)
-}
-
 fn collect_roc_definitions<'a>(object: &object::File<'a, &'a [u8]>) -> MutMap<String, u64> {
     let mut vaddresses = MutMap::default();
 
@@ -1088,7 +1069,7 @@ fn gen_macho_le(
             }
         }
 
-        offset += dbg!(cmd_size);
+        offset += cmd_size;
     }
 
     // cmd_loc should be where the last offset ended

crates/linker/src/pe.rs

@@ -11,7 +11,7 @@ use object::{
         ImageSectionHeader, ImageThunkData64,
     },
     read::pe::ImportTable,
-    LittleEndian as LE, Object, RelocationTarget, SectionIndex,
+    LittleEndian as LE, Object, ObjectSection, ObjectSymbol, RelocationTarget, SectionIndex,
 };
 use serde::{Deserialize, Serialize};
 
@@ -20,7 +20,7 @@ use roc_error_macros::internal_error;
 
 use crate::{
     generate_dylib::APP_DLL, load_struct_inplace, load_struct_inplace_mut,
-    load_structs_inplace_mut, open_mmap, open_mmap_mut,
+    load_structs_inplace_mut, open_mmap, open_mmap_mut, util::is_roc_definition,
 };
 
 /// The metadata stores information about/from the host .exe because
@@ -95,8 +95,6 @@ impl PeMetadata {
     }
 
     fn from_preprocessed_host(preprocessed_data: &[u8], new_sections: &[[u8; 8]]) -> Self {
-        use object::ObjectSection;
-
         let dynhost_obj = object::read::pe::PeFile64::parse(preprocessed_data)
             .unwrap_or_else(|err| internal_error!("Failed to parse executable file: {}", err));
 
@@ -183,17 +181,23 @@ pub(crate) fn preprocess_windows(
     host_exe_filename: &Path,
     metadata_filename: &Path,
     preprocessed_filename: &Path,
-    dummy_dll_symbols: &[String],
+    shared_lib: &Path,
     _verbose: bool,
     _time: bool,
 ) -> object::read::Result<()> {
-    let data = open_mmap(host_exe_filename);
+    let exec_data = open_mmap(host_exe_filename);
+    let shared_lib_data = &*open_mmap(shared_lib);
+    let shared_lib_obj = match object::File::parse(shared_lib_data) {
+        Ok(obj) => obj,
+        Err(e) => internal_error!("Failed to parse shared library file: {e}"),
+    };
+    let dummy_dll_symbols = shared_lib_obj.symbols().filter(is_roc_definition).count();
 
     let new_sections = [*b".text\0\0\0", *b".rdata\0\0"];
     let mut preprocessed = Preprocessor::preprocess(
         preprocessed_filename,
-        &data,
-        dummy_dll_symbols.len(),
+        &exec_data,
+        dummy_dll_symbols,
         &new_sections,
     );
 
@@ -1164,8 +1168,6 @@ fn process_internal_relocations(
 
 impl<'a> AppSections<'a> {
     fn from_data(data: &'a [u8]) -> Self {
-        use object::ObjectSection;
-
         let file = object::File::parse(data).unwrap();
 
         let mut sections = Vec::new();
@@ -1193,8 +1195,6 @@ impl<'a> AppSections<'a> {
             for (offset_in_section, relocation) in section.relocations() {
                 match relocation.target() {
                     RelocationTarget::Symbol(symbol_index) => {
-                        use object::ObjectSymbol;
-
                         let symbol = file.symbol_by_index(symbol_index);
 
                         let address = symbol.as_ref().map(|s| s.address()).unwrap_or_default();
@@ -1252,8 +1252,6 @@ impl<'a> AppSections<'a> {
         let mut other_symbols = Vec::new();
 
         for symbol in file.symbols() {
-            use object::ObjectSymbol;
-
             if symbol.name_bytes().unwrap_or_default().starts_with(b"roc") {
                 if let object::SymbolSection::Section(index) = symbol.section() {
                     let (kind, offset_in_host_section) = section_starts[&index];
@@ -1801,7 +1799,7 @@ mod test {
             &dir.join("host.exe"),
             &dir.join("metadata"),
             &preprocessed_host_filename,
-            &names,
+            &dir.join("libapp.dll"),
             false,
             false,
         )