Run integration tests in default (puma#1394)

* Run integration tests in default

* Add retry in CI

* Convert integration test to ruby script for Windows

* Attempt to add finicky tests back

* Possible kinda janky solution

* Only run master branch on travis (syntax fix)

* Don't do integration tests on windows

Can't be bothered to make those tests crossplatform

* Remove join timeouts on read, let the main test timeout do it

* Assert once

* Sigh.

.travis.yml

@@ -9,7 +9,8 @@ before_install:
   - gem install bundler -v 1.11.2
 script: "TESTOPTS=-v bundle exec rake"
 branches:
-  only: [master]
+  only:
+    - "master"
 rvm:
   - 2.2.7
   - 2.3.4

Gemfile

@@ -7,6 +7,7 @@ gem "rake-compiler"
 
 gem "rack", "< 3.0"
 gem "minitest", "~> 5.9"
+gem "minitest-retry"
 
 gem "jruby-openssl", :platform => "jruby"
 

Rakefile

@@ -3,12 +3,11 @@ require "rake/testtask"
 require "rake/extensiontask"
 require "rake/javaextensiontask"
 require "rubocop/rake_task"
+require 'puma/detect'
 
 # Add rubocop task
 RuboCop::RakeTask.new
 
-IS_JRUBY = defined?(RUBY_ENGINE) ? RUBY_ENGINE == "jruby" : false
-
 spec = Gem::Specification.load("puma.gemspec")
 
 desc "Print the current changelog."
@@ -75,32 +74,21 @@ file 'ext/puma_http11/org/jruby/puma/Http11Parser.java' => ['ext/puma_http11/htt
 end
 task :ragel => ['ext/puma_http11/org/jruby/puma/Http11Parser.java']
 
-if !IS_JRUBY
-
-# compile extensions using rake-compiler
-# C (MRI, Rubinius)
-Rake::ExtensionTask.new("puma_http11", spec) do |ext|
-  # place extension inside namespace
-  ext.lib_dir = "lib/puma"
-
-  ext.cross_compile = true
-  ext.cross_platform = ['i386-mswin32-60', 'i386-mingw32']
-  ext.cross_compiling do |spec|
-    # add fat-binary stub only when cross compiling
-    spec.files << "lib/puma/puma_http11.rb"
-  end
-
-  CLEAN.include "lib/puma/{1.8,1.9}"
-  CLEAN.include "lib/puma/puma_http11.rb"
-end
+if !Puma.jruby?
+  # compile extensions using rake-compiler
+  # C (MRI, Rubinius)
+  Rake::ExtensionTask.new("puma_http11", spec) do |ext|
+    # place extension inside namespace
+    ext.lib_dir = "lib/puma"
 
+      CLEAN.include "lib/puma/{1.8,1.9}"
+      CLEAN.include "lib/puma/puma_http11.rb"
+    end
 else
-
-# Java (JRuby)
-Rake::JavaExtensionTask.new("puma_http11", spec) do |ext|
-  ext.lib_dir = "lib/puma"
-end
-
+  # Java (JRuby)
+  Rake::JavaExtensionTask.new("puma_http11", spec) do |ext|
+    ext.lib_dir = "lib/puma"
+  end
 end
 
 # the following is a fat-binary stub that will be used when
@@ -116,7 +104,7 @@ end
 Rake::TestTask.new(:test)
 
 # tests require extension be compiled, but depend on the platform
-if IS_JRUBY
+if Puma.jruby?
   task :test => [:java]
 else
   task :test => [:compile]
@@ -135,15 +123,15 @@ end
 namespace :test do
   desc "Run the integration tests"
   task :integration do
-    sh "cd test/shell; sh run.sh"
+    sh "ruby test/shell/run.rb"
   end
 
   desc "Run all tests"
-  if defined?(JRUBY_VERSION) and ENV['TRAVIS']
+  if (Puma.jruby? && ENV['TRAVIS']) || Puma.windows?
     task :all => :test
   else
     task :all => [:test, "test:integration"]
   end
 end
 
-task :default => [:rubocop, :test]
+task :default => [:rubocop, "test:all"]

gemfiles/2.1-Gemfile

@@ -10,6 +10,7 @@ gem "rake-compiler"
 
 gem "rack", "~> 1.6"
 gem "minitest", '~> 5.9'
+gem "minitest-retry"
 
 gem "jruby-openssl", :platform => "jruby"
 

test/helper.rb

@@ -1,7 +1,10 @@
 # Copyright (c) 2011 Evan Phoenix
 # Copyright (c) 2005 Zed A. Shaw
 
-require 'stopgap_13632' if %w(2.2.7 2.3.4 2.4.1).include? RUBY_VERSION
+begin
+  require 'stopgap_13632' if %w(2.2.7 2.3.4 2.4.1).include? RUBY_VERSION
+rescue LoadError
+end
 
 begin
   require "bundler/setup"
@@ -43,15 +46,16 @@ class TestTookTooLong < Timeout::Error
   end
 
   def run(*)
-    if ENV['CI']
-      ::Timeout.timeout(Puma.jruby? ? 120 : 30, TestTookTooLong) { super }
-    else
-      super # we want to be able to use debugger
-    end
+    ::Timeout.timeout(Puma.jruby? ? 120 : 60, TestTookTooLong) { super }
   end
 end
 
-Minitest::Test.prepend TimeoutEveryTestCase
+if ENV['CI']
+  Minitest::Test.prepend TimeoutEveryTestCase
+
+  require 'minitest/retry'
+  Minitest::Retry.use!
+end
 
 module SkipTestsBasedOnRubyEngine
   def skip_on_jruby

test/shell/run.rb

@@ -0,0 +1,9 @@
+results = %w[t1 t2 t3].map do |test|
+  system("ruby -rubygems test/shell/#{test}.rb ") # > /dev/null 2>&1
+end
+
+if results.any? { |r| r != true }
+  exit 1
+else
+  exit 0
+end

test/shell/t1.rb

@@ -1,4 +1,4 @@
-system "ruby -rubygems -I../../lib ../../bin/puma -p 10102 -C t1_conf.rb ../rackup/hello.ru &"
+system "ruby -rubygems -Ilib bin/puma -p 10102 -C test/shell/t1_conf.rb test/rackup/hello.ru &"
 sleep 5
 system "curl http://localhost:10102/"
 

test/shell/t2.rb

@@ -1,8 +1,8 @@
-system "ruby -rubygems -I../../lib ../../bin/pumactl -F t2_conf.rb start"
+system "ruby -rubygems -Ilib bin/pumactl -F test/shell/t2_conf.rb start"
 sleep 5
 system "curl http://localhost:10103/"
 
-system "ruby -rubygems -I../../lib ../../bin/pumactl -F t2_conf.rb stop"
+system "ruby -rubygems -Ilib bin/pumactl -F test/shell/t2_conf.rb stop"
 
 sleep 1
 

test/shell/t3.rb

@@ -1,4 +1,4 @@
-system "ruby -rubygems -I../../lib ../../bin/puma -p 10102 -C t3_conf.rb ../rackup/hello.ru &"
+system "ruby -rubygems -Ilib bin/puma -p 10102 -C test/shell/t3_conf.rb test/rackup/hello.ru &"
 sleep 5
 
 worker_pid_was_present = File.file? "t3-worker-2-pid"

test/test_integration.rb

@@ -146,48 +146,54 @@ def test_stop_via_pumactl
 
     ccli.run
 
-    assert_kind_of Thread, t.join(1), "server didn't stop"
+    assert_kind_of Thread, t.join, "server didn't stop"
   end
 
   def test_phased_restart_via_pumactl
-    skip "Too finicky, fails 50% of the time on CI" if ENV["CI"]
-    skip if Puma.jruby? || Puma.windows?
+    skip if Puma.jruby? || Puma.windows? || ENV['CI']
 
-    launcher_thread = Thread.new do
-      conf = Puma::Configuration.new do |c|
-        c.quiet
-        c.state_path @state_path
-        c.bind "unix://#{@bind_path}"
-        c.activate_control_app "unix://#{@control_path}", :auth_token => @token
-        c.workers 2
-        c.worker_shutdown_timeout 1
-        c.rackup "test/rackup/hello-stuck.ru"
-      end
-      Puma::Launcher.new(conf, :events => @events).run
+    conf = Puma::Configuration.new do |c|
+      c.quiet
+      c.state_path @state_path
+      c.bind "unix://#{@bind_path}"
+      c.activate_control_app "unix://#{@control_path}", :auth_token => @token
+      c.workers 2
+      c.worker_shutdown_timeout 1
+      c.rackup "test/rackup/hello-stuck.ru"
+    end
+
+    l = Puma::Launcher.new conf, :events => @events
+
+    t = Thread.new do
+      Thread.current.abort_on_exception = true
+      l.run
     end
 
     wait_booted
 
-    # Make both workers stuck
-    s1 = UNIXSocket.new @bind_path
-    s1 << "GET / HTTP/1.0\r\n\r\n"
+    s = UNIXSocket.new @bind_path
+    s << "GET / HTTP/1.0\r\n\r\n"
 
     sout = StringIO.new
-
     # Phased restart
     ccli = Puma::ControlCLI.new ["-S", @state_path, "phased-restart"], sout
     ccli.run
-    sleep 20
-    @events.stdout.rewind
-    log = @events.stdout.readlines.join("")
-    assert_match(/TERM sent/, log)
-    assert_match(/- Worker \d \(pid: \d+\) booted, phase: 1/, log)
 
+    done = false
+    until done
+      @events.stdout.rewind
+      log = @events.stdout.readlines.join("")
+      if log.match(/- Worker \d \(pid: \d+\) booted, phase: 1/)
+        assert_match(/TERM sent/, log)
+        assert_match(/- Worker \d \(pid: \d+\) booted, phase: 1/, log)
+        done = true
+      end
+    end
     # Stop
     ccli = Puma::ControlCLI.new ["-S", @state_path, "stop"], sout
     ccli.run
 
-    assert_kind_of Thread, launcher_thread.join(5), "server didn't stop"
+    assert_kind_of Thread, t.join, "server didn't stop"
   end
 
   def test_kill_unknown_via_pumactl

test/test_puma_server.rb

@@ -296,8 +296,6 @@ def test_status_hook_fires_when_server_changes_states
   end
 
   def test_timeout_in_data_phase
-    skip("Hangs too often, TODO: fix")
-
     @server.first_data_timeout = 2
     @server.add_tcp_listener @host, @port
     @server.run