Build script cleanup

This commit attempts to clean up the code responsible for compiling Ruby
in the build script, as well as get the crate compiling on Windows.

For Unix, the main change here is that I've removed any attempts at
skipping steps beyond those built into `make`. This should still skip
the most expensive parts of the build, but be much more resilient. It is
possible (or even likely) that this is going to be slightly slower than
what was there before. If this is undesirable, we should move the C
build out to a separate crate as outlined in fables-tales#310.

On Windows, this goes from the crate not compiling to the crate
compiling. This commit does *not* mean that the build on Windows is
working, yet. On my machine, it builds but immediately segfaults when
run. This is most likely because I have configured Ruby to skip all
extensions except ripper. The most likely candidate is `enc` (which
unfortunately is also the one I was struggling to get building), but I
haven't dug in further.

Even though Windows builds segfault, I don't think that should block
this PR being merged. I've spent several hours banging my head against
this, and need to walk away from it for a while. The fact that it
*compiles* is still a step in the right direction. The fact that this is
strictly an improvement on what was happening before is why I opted to
submit this in this state.

The way Windows build tools are set up is... Funky. MSVC intends you to
pretty much always build through Visual Studio, and if you install just
the build tools you're expected to run through a shortcut it provides
which links MSVC DLLs and sets a ton of env vars for you. This is
definitely suboptimal, and I couldn't get the crate to build at all when
run from the MSVC console.

The CC crate seems to handle most of this for us. I still ran into a few
issues figuring everything out (Ruby builds in 32 bit by default even
when compiled on a 64 bit system for... some reason), and I couldn't
some extensions (enc and bigdecimal, possibly others) on 64-bit. But I
don't think that's because of the build environment.

The other changes were pretty straight forward. There was one instance
where we assumed `long` was 64 bits (which isn't true), and the
jemallocator crate doesn't build on Windows at the moment, so I've
disabled it.

The winapi dependency is basically just to get the appropriate
`rustc-link-libs` that are needed for Ruby. I'm not sure if the
`everything` feature actually slows anything down at runtime or not. It
should probably get pared down at some point, but after realizing it's
at least 6 or 7 dylibs I punted this for now

Fixes fables-tales#301.
Related to fables-tales#304.
Related to fables-tales#310.

librubyfmt/Cargo.toml

@@ -12,12 +12,17 @@ serde = { version = "1.0", features=["derive"] }
 serde_json = "1.0.40"
 bytecount = "0.6.0"
 backtrace = "0.3.45"
-jemallocator = { version = "0.3.0", features = ["disable_initial_exec_tls"], optional=true }
 libc = "0.2.68"
 ripper_deserialize = { path = "ripper_deserialize" }
 log = { version = "0.4.8", features = ["max_level_debug", "release_max_level_warn"] }
 simplelog = "0.8"
 
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+jemallocator = { version = "0.3.0", features = ["disable_initial_exec_tls"], optional=true }
+
+[target.'cfg(windows)'.dependencies]
+winapi = { version = "0.3", features = ["everything"] }
+
 [build-dependencies]
 cc = "1.0"
 

librubyfmt/build.rs

@@ -1,74 +1,135 @@
-use std::io::{self, Write};
-use std::process::Command;
+#[cfg(windows)]
+use std::env;
+use std::error::Error;
+use std::path::Path;
+use std::process::{Command, ExitStatus};
 
-fn main() {
+type Output = Result<(), Box<dyn Error>>;
+
+fn main() -> Output {
     #[cfg(target_os = "linux")]
     let libname = "ruby-static";
     #[cfg(target_os = "macos")]
     let libname = "ruby.2.6-static";
+    #[cfg(all(target_arch = "x86_64", windows))]
+    let libname = "x64-vcruntime140-ruby260-static";
+    #[cfg(all(target_arch = "x86", windows))]
+    let libname = "vcruntime140-ruby260-static";
+    #[cfg(all(target_env = "gnu", windows))]
+    compile_error!("rubyfmt on Windows is currently only supported with msvc");
 
-    let path = std::env::current_dir().expect("is current");
-    let ruby_checkout_path = path.join("ruby_checkout/ruby-2.6.6/");
-    if !ruby_checkout_path
-        .join(format!("lib{}.a", libname))
-        .exists()
-    {
-        let o = Command::new("bash")
-            .arg("-c")
-            .arg(format!(
-                "autoconf && {}/configure --without-gmp --disable-jit-support && make -j",
-                ruby_checkout_path.display()
-            ))
-            .current_dir(&ruby_checkout_path)
-            .output()
-            .expect("works1 ");
-        if !o.status.success() {
-            io::stdout().write_all(&o.stdout).unwrap();
-            io::stderr().write_all(&o.stderr).unwrap();
-            panic!("failed subcommand");
-        }
-    }
-    if !ruby_checkout_path.join("libripper.2.6-static.a").exists() {
-        let o = Command::new("bash")
-            .arg("-c")
-            .arg("ar crus libripper.2.6-static.a ext/ripper/ripper.o")
-            .current_dir(&ruby_checkout_path)
-            .output()
-            .expect("works");
-        if !o.status.success() {
-            panic!("failed subcommand");
-        }
-    }
-
+    let path = std::env::current_dir()?;
+    let ruby_checkout_path = path.join("ruby_checkout").join("ruby-2.6.6");
+    make_configure(&ruby_checkout_path)?;
+    run_configure(&ruby_checkout_path)?;
+    build_ruby(&ruby_checkout_path)?;
+    #[cfg(unix)]
+    let ripper = "ext/ripper/ripper.o";
+    #[cfg(windows)]
+    let ripper = "ext/ripper/ripper.obj";
     cc::Build::new()
         .file("src/rubyfmt.c")
-        .include(format!("{}/include", ruby_checkout_path.display()))
-        .include(format!(
-            "{}/.ext/include/x86_64-darwin20",
-            ruby_checkout_path.display()
-        ))
-        .include(format!(
-            "{}/.ext/include/x86_64-darwin19",
-            ruby_checkout_path.display()
-        ))
-        .include(format!(
-            "{}/.ext/include/x86_64-darwin18",
-            ruby_checkout_path.display()
-        ))
-        .include(format!(
-            "{}/.ext/include/x86_64-linux",
-            ruby_checkout_path.display()
-        ))
-        .compile("librubyfmt_c");
+        .object(ruby_checkout_path.join(&ripper))
+        .include(ruby_checkout_path.join("include"))
+        .include(ruby_checkout_path.join(".ext/include/x86_64-darwin20"))
+        .include(ruby_checkout_path.join(".ext/include/x86_64-darwin19"))
+        .include(ruby_checkout_path.join(".ext/include/x86_64-darwin18"))
+        .include(ruby_checkout_path.join(".ext/include/x86_64-linux"))
+        .include(ruby_checkout_path.join(".ext/include/x64-mswin64_140"))
+        .include(ruby_checkout_path.join(".ext/include/i386-mswin32_140"))
+        .compile("rubyfmt_c");
 
     println!(
-        "cargo:rustc-link-search=native={}/ruby_checkout/ruby-2.6.6",
-        path.display()
+        "cargo:rustc-link-search=native={}",
+        ruby_checkout_path.display()
     );
     println!("cargo:rustc-link-lib=static={}", libname);
-    println!("cargo:rustc-link-lib=static=ripper.2.6-static");
+    #[cfg(not(windows))]
     println!("cargo:rustc-link-lib=dylib=z");
 
     #[cfg(target_os = "linux")]
     println!("cargo:rustc-link-lib=dylib=crypt");
+
+    Ok(())
+}
+
+#[cfg(unix)]
+fn make_configure(ruby_checkout_path: &Path) -> Output {
+    if ruby_checkout_path.join("Makefile").exists() {
+        let o = Command::new("make")
+            .arg("configure")
+            .current_dir(ruby_checkout_path)
+            .status()?;
+        check_process_success("make configure", o)
+    } else {
+        let o = Command::new("autoconf")
+            .current_dir(ruby_checkout_path)
+            .status()?;
+        check_process_success("autoconf", o)
+    }
+}
+
+#[cfg(windows)]
+fn make_configure(_: &Path) -> Output {
+    Ok(())
+}
+
+#[cfg(unix)]
+fn run_configure(ruby_checkout_path: &Path) -> Output {
+    let o = Command::new("./configure")
+        .arg("--without-gmp")
+        .arg("--disable-jit-support")
+        .current_dir(ruby_checkout_path)
+        .status()?;
+    check_process_success("./configure", o)
+}
+
+#[cfg(windows)]
+fn run_configure(ruby_checkout_path: &Path) -> Output {
+    let mut command = Command::new(ruby_checkout_path.join("win32/configure.bat"));
+    command
+        .arg("--without-gmp")
+        .arg("--disable-mjit-support")
+        .arg("--with-static-linked-ext")
+        .arg("--disable-install-doc")
+        .arg("--with-ext=ripper")
+        .envs(find_tool("nmake.exe")?.env().iter().cloned())
+        .current_dir(ruby_checkout_path);
+    #[cfg(target_arch = "x86_64")]
+    command.arg("--target=x64-mswin64");
+    let o = command.status()?;
+    check_process_success("win32/configure.bat", o)
+}
+
+#[cfg(unix)]
+fn build_ruby(ruby_checkout_path: &Path) -> Output {
+    let o = Command::new("make")
+        .arg("-j")
+        .current_dir(ruby_checkout_path)
+        .status()?;
+    check_process_success("make", o)
+}
+
+#[cfg(windows)]
+fn build_ruby(ruby_checkout_path: &Path) -> Output {
+    let o = find_tool("nmake.exe")?
+        .to_command()
+        .current_dir(ruby_checkout_path)
+        .status()?;
+    check_process_success("nmake", o)
+}
+
+#[cfg(windows)]
+fn find_tool(tool: &str) -> Result<cc::Tool, Box<dyn Error>> {
+    let target = env::var("TARGET")?;
+    cc::windows_registry::find_tool(&target, tool)
+        .ok_or_else(|| format!("Failed to find {}", tool).into())
+}
+
+fn check_process_success(command: &str, code: ExitStatus) -> Output {
+    if code.success() {
+        Ok(())
+    } else {
+        Err(format!("Command {} failed with: {}", command, code).into())
+    }
 }

librubyfmt/src/lib.rs

@@ -6,7 +6,7 @@ use std::io::{Cursor, Write};
 use std::slice;
 use std::str;
 
-#[cfg(feature = "use_jemalloc")]
+#[cfg(all(feature = "use_jemalloc", not(target_env = "msvc")))]
 #[global_allocator]
 static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
 

librubyfmt/src/ruby_ops.rs

@@ -35,7 +35,7 @@ impl Parser {
 
     pub fn new(buf: &str) -> Self {
         unsafe {
-            let buffer_string = rb_utf8_str_new(buf.as_ptr() as _, buf.len() as i64);
+            let buffer_string = rb_utf8_str_new(buf.as_ptr() as _, buf.len() as libc::c_long);
             let parser_class = rb_const_get_at(rb_cObject, intern!("Parser"));
             let parser_instance = rb_funcall(parser_class, intern!("new"), 1, buffer_string);
             Parser(parser_instance)