Attempt to make harness more robust

The most important change here is that we add the ↵ test type, which
expects a CR or LF ahead of its text to match.  Ideally we'd assert
this every time, but there's a variety of extraneous stuff shells like
to print that can interfere with this.

I've also tweaked the timing a bit, trying to find a balance between
having the suite run quickly, while accommodating slower machines and
other sources of inconsistency.  I'm sure there will still be more
tweaks.

Fixes #10.

helpers/harness.tcl

@@ -19,9 +19,11 @@
 # ☠ ⇒ shell itself exits, with code
 # → ⇒ user input
 # ← ⇒ response on stdout
+# ↵ ⇒ response on stdout delimited by a CR or LF
 # ≠ ⇒ anything but this on stdout (at least one line)
 # ✓ ⇒ expect zero exit status of previous command (not implemented)
 # ✗ ⇒ expect nonzero exit status of previous command (not implemented)
+# ⌛ ⇒ wait a little extra (0.5 seconds)
 
 proc expand {s} {
     string map {
@@ -68,7 +70,11 @@ proc setup_execution_environment {} {
 }
 
 proc wait_for_exit {} {
-    expect eof;         # seems to be needed for buffering issues on BSD
+    # seems to be needed for buffering issues on BSD
+    expect {
+        eof {}
+        timeout {}
+    }
     foreach {pid spawn_id is_os_error code} [wait] break
     if {0 == $is_os_error} { return $code }
     not_ok
@@ -88,21 +94,20 @@ fconfigure $test_file -encoding utf-8
 set n_tests 0
 while {![eof $test_file]} {
     switch -re [string index [gets $test_file] 0] {
-        ←|≠|✓|✗|☠ {incr n_tests}
+        ←|↵|≠|✓|✗|☠ {incr n_tests}
         default {}
     }
 }
 seek $test_file 0 start
 
 log_user 0
-set send_slow { 1 .01 }
 setup_execution_environment
 if [catch {spawn $shell} err] {error $err}
-# waiting for first prompt can help
+# waiting a little extra for first prompt can help.  if your shell
+# emits no prompt, that's not very friendly.
+set timeout 3
+expect -re .+
 set timeout 2
-expect -re .
-expect *
-set timeout 1
 
 set line_num 0
 set test_num 1
@@ -124,22 +129,44 @@ proc not_ok {} {
 }
 proc is {x} { uplevel 1 [list if $x {ok} {not_ok}]}
 
+proc re_quote {s} {
+    string map {
+        \\ \\\\
+        ^ \\^
+        \$ \\\$
+        \[ \\\[
+        \( \\\(
+        . \\.
+        + \\+
+        * \\*
+        ? \\?
+        \{ \\\{
+    } $s
+}
+
 proc expect_line {line} {
     # NB: this is fairly unreliable, and we should have some lint
     # where we reject tests whose expected output would match the tail
     # of their input.  There doesn't seem to be a better way to deal
     # with the output and timing differences between shells, short of
     # going full ptrace as mentioned previously.
     set rv [expect {
-        "$line\r\n" {return 1}
+        -re $line {return 1}
         default {return 0}
     }]
     expect *
     return $rv
 }
 
+# The timing of sending a line can be tricky.  Shells are usually
+# sending their prompts to stderr, and so output to stdout can race
+# with that.  We used to use send -s (send_slow) here, but in practice
+# it seems unnecessary; what does seem necessary, is giving the shell
+# a little bit of time before we look at the output or send the next
+# line.
 proc careful_send {m} {
-    if {[catch {send -s $m} err]} { error $err }
+    if {[catch {send -- $m} err]} { error $err }
+    sleep 0.01
 }
 
 if {1 == $emit_tap} {puts "1..$n_tests"}
@@ -149,11 +176,13 @@ while {![eof $test_file]} {
     if {0 == [string length [string trim $command]]} { continue }
     set line [expand [rest-of $command]]
     switch [string index $command 0] {
-        → {expect *; careful_send $line; sleep 0.1}
-        ← {is {1 == [expect_line $line]}}
-        ≠ {is {0 == [expect_line $line]}}
+        → {expect *; careful_send $line}
+        ↵ {is {1 == [expect_line "\[\r\n][re_quote $line]"]}}
+        ← {is {1 == [expect_line [re_quote $line]]}}
+        ≠ {is {0 == [expect_line "\[\r\n][re_quote $line]"]}}
         ✓ {error "sorry we decided not to do this $line_num"}
         ✗ {error "sorry we decided not to do this $line_num"}
+        ⌛ {sleep 0.2}
         ☠ {
             is {$line eq [wait_for_exit]}
             if {$test_num <= $n_tests} {
@@ -168,5 +197,5 @@ while {![eof $test_file]} {
 }
 
 # Calling close causes zsh to exit unhappily, so we send ^D instead.
-send -s "\x04"
+sleep 0.1; send "\x04"
 if {0 != [wait_for_exit]} {error "shell didn't exit cleanly"}

stage_1/01-fork-exec.t

@@ -1,8 +1,10 @@
 → /bin/echo foo bar⏎
-← foo bar
-→ tr a-z n-za-m⏎foo bar baz⏎^D
-← sbb one onm
+↵ foo bar
+→ tr a-z n-za-m⏎
+→ foo bar baz⏎
+↵ sbb one onm
+→ ^D
 → ./helpers/echo-rot13 foo bar⏎
-← sbb one
+↵ sbb one
 → ./helpers/echo-exit ./helpers/successful-exit-status⏎
-← 0
+↵ 0

stage_1/02-commands-invoked-from-path.t

@@ -1,3 +1,3 @@
 # harness puts helpers/ in our path
 → echo-exit ./helpers/successful-exit-status⏎
-← 0
+↵ 0

stage_1/03-cd-changes-directory.t

@@ -1,6 +1,6 @@
 → /bin/pwd⏎
-≠ /tmp
+≠ /tmp\n
 → cd /tmp⏎
 → cd ..⏎
 → /bin/pwd⏎
-← /
+↵ /\n

stage_1/04-semicolon-separates-commands.t

@@ -1,7 +1,7 @@
 # POSIX calls these sequential lists
 → echo -n foo; echo -n bar; echo baz⏎
-← foobarbaz
+↵ foobarbaz
 → cd /tmp; pwd⏎
-← /tmp
+↵ /tmp
 → false; echo-rot13 foo⏎
-← sbb
+↵ sbb

stage_1/05-and-or-chaining.t

@@ -1,15 +1,19 @@
 # POSIX calls these AND Lists and OR Lists
 → true && echo-rot13 foo⏎
-← sbb
+↵ sbb
 → false && echo-rot13 foo⏎
 ≠ sbb
 → true && false && echo-rot13 foo⏎
 ≠ sbb
 → false || echo-rot13 foo⏎
-← sbb
+↵ sbb
 → true || false || echo-rot13 foo⏎
 ≠ sbb
 → false || true && echo-rot13 foo⏎
-← sbb
+↵ sbb
+# we wait a bit here because some shells have an expensive
+# command_not_found_handler hook; e.g. Fedora checks a package
+# database.
 → nonexistent-command || echo-rot13 zim⏎
-← mvz
+⌛
+↵ mvz

stage_1/07-subshell.t

@@ -1,10 +1,10 @@
 → pwd⏎
-≠ /tmp
+≠ /tmp\n
 → (cd /tmp; pwd)⏎
-← /tmp
+↵ /tmp\n
 → pwd⏎
-≠ /tmp
+≠ /tmp\n
 → (exit 1) && echo-rot13 foo⏎
-≠ sbb
+≠ sbb\n
 → (exit 0 && exit 1) && echo-rot13 foo⏎
-← sbb
+↵ sbb\n

stage_1/08-bang-negates-exit-code.t

@@ -1,6 +1,6 @@
 → ! false && echo-rot13 foo⏎
-← sbb
+↵ sbb
 → ! true && echo-rot13 foo⏎
 ≠ sbb
 → ! true || echo-rot13 foo⏎
-← sbb
+↵ sbb

stage_1/09-backslash-continues-line.t

@@ -1,2 +1,2 @@
 → echo foo\⏎bar\⏎baz⏎
-← foobarbaz
+← foobarbaz\n

stage_1/10-multiline-list.t

@@ -1,2 +1,5 @@
-→ true &&⏎false ||⏎echo-rot13 foo ||⏎echo-rot13 bar⏎
-← sbb
+→ true &&⏎
+→ false ||⏎
+→ echo-rot13 foo ||⏎
+→ echo-rot13 bar⏎
+↵ sbb

stage_2/01-pipeline-execution.t

@@ -1,2 +1,2 @@
 → echo foo | grep -c foo⏎
-← 1
+↵ 1

stage_2/02-redirections.t

@@ -1,22 +1,22 @@
 # we're started in a temp directory
 → echo zim > foo && cat foo | tr n-za-m a-z⏎
-← mvz
+↵ mvz
 → cat foo⏎
-← zim
+↵ zim
 → echo a > foo ; echo b > foo ; cat foo⏎
-← b
+↵ b
 # NB echo -n isn't portable...
 → echo -n a > foo ; echo b >> foo ; cat foo⏎
-← ab
+↵ ab
 → cat < /non-existent-file || echo-rot13 foo⏎
-← sbb
+↵ sbb
 → cat /non-existent-file 2>foo || grep -c . <foo⏎
-← 1
+↵ 1
 → echo quux > foo && cat < foo⏎
-← quux
+↵ quux
 → echo quux >foo && echo -n blah <>foo && cat <foo⏎
-← blahquux
+↵ blahquux
 → echo bar >foo && cat <>foo⏎
-← bar
+↵ bar
 → echo mitten >foo && echo -n k 1<>foo && cat foo⏎
-← kitten
+↵ kitten

stage_2/03-redirection-permissions.t

@@ -1,15 +1,15 @@
 # NB stdin/out/err when connected to a terminal is usually rw
 → fd-perms 0 </dev/null⏎
-← r
+↵ r
 → fd-perms 0 <>/dev/null⏎
-← rw
+↵ rw
 # it's tricky to test fd 1 without losing the output.  we could have
 # an exit status instead, with tools called is-writable and
 # is-readable, but meh.
 → fd-perms 3 3>/dev/null⏎
-← w
+↵ w
 # An Arrow in Heart
 → fd-perms 3 3</dev/null⏎
-← r
+↵ r
 → fd-perms 3 3<>/dev/null⏎
-← rw
+↵ rw

stage_2/04-ensure-pipes-are-closed.t

@@ -1,4 +1,4 @@
 # This will hang in the case of a very common bug: you forget to close
 # the other ends of the pipe.
 → cat </etc/passwd | cat | cat | cat >/dev/null | echo-rot13 foo⏎
-← sbb
+↵ sbb

stage_2/05-no-extraneous-fds.t

@@ -1,8 +1,8 @@
 → list-fds⏎
-← 0\n1\n2
+↵ 0\n1\n2
 # probably want to check a few other cases of this
 → list-fds 0<&- 2<&-⏎
-← 1
+↵ 1
 # is this order specified by POSIX?  seems like you could get 0,1 here
 → list-fds 0<&- 2<&- 3</dev/null⏎
-← 1\n3
+↵ 1\n3

stage_2/06-process-substitution.t

@@ -1,2 +1,2 @@
-→ echo foo > bar && cat $(echo bar)⏎
-← foo
+→ echo-rot13 foo > bar && cat $(echo bar)⏎
+↵ sbb

stage_2/07-bang-only-at-start-of-pipelines.t

@@ -1,5 +1,5 @@
-# Would prefer to test this but mksh doesn't agree
-#→ echo zim | ! tr z b | grep -qc bim || echo foo⏎
-#≠ foo
+# mksh does not agree with this interpretation
+→ echo zim | ! tr z b | grep -qc bim || echo foo⏎
+≠ foo
 → ! echo zim | tr z b | grep -qc bim || echo-rot13 foo⏎
-← sbb
+↵ sbb

stage_2/08-builtin-in-pipeline.t

@@ -1,4 +1,4 @@
 → echo foo | cd /tmp | pwd⏎
-≠ /tmp
+≠ /tmp\n
 → echo foo | cd /tmp | echo-rot13 bar⏎
-← one
+↵ one

stage_2/process-redirection.bonus

@@ -1,3 +1,3 @@
 # not part of posix shell
 → cat <(echo foo)
-← foo
+⏎ foo

stage_3/01-executed-command-receives-signals.t

@@ -1,9 +1,11 @@
 # this would be nicer if we already had $@...
 → echo-signal INT⏎
-← ready
-→ ^C⏎
+↵ ready
+→ ^C
+⌛
 ← INT
 → echo-signal TSTP⏎
-← ready
-→ ^Z⏎
+↵ ready
+→ ^Z
+⌛
 ← TSTP

stage_3/02-job-control-with-fg-bg.t

@@ -1,14 +1,16 @@
 → echo-signal CONT⏎
-← ready
+↵ ready
 → ^Z
+⌛
 → echo-rot13 foo⏎
-← sbb
+↵ sbb
 → fg⏎
 ← CONT
 → echo-signal INT &⏎
 ← ready
 → echo-rot13 foo⏎
-← sbb
+↵ sbb
 → fg⏎
-→ ^C⏎
+⌛
+→ ^C
 ← INT

stage_3/03-sequencing-with-suspend.t

@@ -1,6 +1,8 @@
-→ echo-signal INT & sleep 1; echo-rot13 foo⏎
+→ echo-signal INT & sleep 0.5; echo-rot13 foo⏎
 ← ready
+⌛
 ← sbb
 → fg⏎
-→ ^C⏎
+⌛
+→ ^C
 ← INT

stage_3/04-ttin-and-ttou.t

@@ -1,10 +1,10 @@
 → tr a-z n-za-m </dev/tty &⏎
 → fg⏎
 → foo⏎
-← sbb
+↵ sbb
 → ^D
 → stty tostop⏎
 → echo foo &⏎
 ≠ foo
 → fg⏎
-← foo
+↵ foo