Allow indentation in REPL

examples/test.pf

@@ -1,31 +1,6 @@
-def 
+def
 
-foo(i int) :
-    "foo"
+qux :
+    SQL --- foo bar
 
 
-zort(t ... any?) :
-    foo +
-
-troz(i, j, k int) :
-    "troz"
-
-var 
-
-T tuple = ()
-U tuple = 1, 2, 3
-V tuple = tuple(1)
-W tuple = tuple("zort")
-X tuple = "a", "b", "c"
-
-x = set(1, 2, 3, 4, 5, 6)
-y = set(2, 3, 4)
-z = set(6, 5, 4)
-
-a = set("a", "b", "c", "d", "e", "f")
-b = set("a", "b", "c")
-c = set("c", "f", "d")
-
-const 
-
-Z tuple = "a", "b", "c"

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 // indirect
 	github.com/aws/smithy-go v1.20.3 // indirect
+	github.com/chzyer/readline v1.5.1 // indirect
 	github.com/coreos/go-oidc/v3 v3.5.0 // indirect
 	github.com/danieljoos/wincred v1.2.2 // indirect
 	github.com/databricks/databricks-sql-go v1.5.7 // indirect

go.sum

@@ -47,6 +47,10 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2 h1:sZXIzO38GZOU+O0C+INqbH7C2yALw
 github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2/go.mod h1:Lcxzg5rojyVPU/0eFwLtcyTaek/6Mtic5B1gJo7e/zE=
 github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE=
 github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
+github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
+github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
+github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
+github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
 github.com/coreos/go-oidc/v3 v3.5.0 h1:VxKtbccHZxs8juq7RdJntSqtXFtde9YpNpGn0yqgEHw=
 github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM=
 github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
@@ -279,6 +283,7 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

source/hub/hub.go

@@ -1494,6 +1494,7 @@ var (
 	GOOD_BULLET    = Green("  ▪ ")
 	BROKEN         = Red("  ✖ ")
 	PROMPT         = "→ "
+	INDENT_PROMPT  = "  "
 	ERROR          = "$Error$"
 	RT_ERROR       = "$Error$"
 	HUB_ERROR      = "$Hub error$"

source/hub/repl.go

@@ -2,13 +2,15 @@ package hub
 
 import (
 	"io"
+	"regexp"
 	"strings"
 
-	"github.com/lmorg/readline"
+	"github.com/chzyer/readline"
 )
 
 func StartHub(hub *Hub, in io.Reader, out io.Writer) {
-	rline := readline.NewInstance()
+	var rline *readline.Instance
+	colonOrEmdash, _ := regexp.Compile(`.*[\w\s]*(:|---)[\s]*$`)
 	for {
 
 		// The hub's CurrentForm setting allows it to ask for information from the user instead of
@@ -19,23 +21,15 @@ func StartHub(hub *Hub, in io.Reader, out io.Writer) {
 
 			for {
 				queryString := hub.CurrentForm.Fields[len(hub.CurrentForm.Result)]
-				// A * at the beginning of the query string indicates that the answer should be
-				// masked.
-				if queryString[0] == '*' {
-					queryString = queryString[1:]
-					rline.PasswordMask = '▪'
-				}
-
 				// The readln utility doesn't like multiline prompts, so we must kludge a little.
 				pos := strings.LastIndex(queryString, "\n")
 				if pos == -1 {
-					rline.SetPrompt(queryString + ": ")
+					rline, _ = readline.New(queryString + ": ")
 				} else {
 					hub.WriteString(queryString[:pos+1])
-					rline.SetPrompt(queryString[pos+1:] + ": ")
+					rline, _ = readline.New(queryString[pos+1:] + ": ")
 				}
 				line, _ := rline.Readline()
-				rline.PasswordMask = 0
 				hub.CurrentForm.Result[hub.CurrentForm.Fields[len(hub.CurrentForm.Result)]] = line
 				if len(hub.CurrentForm.Result) == len(hub.CurrentForm.Fields) {
 					hub.CurrentForm.Call(hub.CurrentForm)
@@ -45,25 +39,52 @@ func StartHub(hub *Hub, in io.Reader, out io.Writer) {
 			continue
 		}
 
-		rline.SetPrompt(makePrompt(hub))
-		line, _ := rline.Readline()
-
-		line = strings.TrimSpace(line)
+		ws := ""
+		input := ""
+		c := 0
+		for {
+			rline, _ = readline.New(makePrompt(hub, ws != ""))
+			line, _ := rline.ReadlineWithDefault(ws)
+			c++
+			input = input + line + "\n"
+			ws = ""
+			for _, c := range line {
+				if c == ' ' || c == '\t' {
+					ws = ws + string(c)
+				} else {
+					break
+				}
+			}
+			if colonOrEmdash.Match([]byte(line)) {
+				ws = ws + "\t"
+			}
+			if ws == "" {
+				break
+			}
+		}
+		input = strings.TrimSpace(input)
 
-		_, quitCharm := hub.Do(line, hub.Username, hub.Password, hub.currentServiceName())
-		if quitCharm {
+		_, quit := hub.Do(input, hub.Username, hub.Password, hub.currentServiceName())
+		if quit {
 			break
 		}
 	}
 }
 
-func makePrompt(hub *Hub) string {
+func makePrompt(hub *Hub, indented bool) string {
+	symbol := PROMPT 
+	left := hub.currentServiceName()
+	if indented {
+		symbol = INDENT_PROMPT
+		left = strings.Repeat(" ", len(left))
+	}
 	if hub.currentServiceName() == "" {
-		return PROMPT
+		return symbol
 	}
-	promptText := hub.currentServiceName() + " " + PROMPT
+	promptText := left + " " + symbol
 	if hub.CurrentServiceIsBroken() {
 		promptText = Red(promptText)
 	}
 	return promptText
 }
+

source/initializer/initializer.go

@@ -334,10 +334,7 @@ func (iz *initializer) MakeParserAndTokenizedProgram() {
 	)
 
 	tok = iz.p.TokenizedCode.NextToken() // note that we've already removed leading newlines.
-	if settings.SHOW_RELEXER && !(settings.IGNORE_BOILERPLATE && settings.ThingsToIgnore.Contains(tok.Source)) {
-		println(tok.Type, tok.Literal)
-	}
-
+
 	if tok.Type == token.EOF { // An empty file should still initiate a service, but one with no data.
 		return
 	}
@@ -352,9 +349,6 @@ func (iz *initializer) MakeParserAndTokenizedProgram() {
 	line := token.NewCodeChunk()
 
 	for tok = iz.p.TokenizedCode.NextToken(); tok.Type != token.EOF; tok = iz.p.TokenizedCode.NextToken() {
-		if settings.SHOW_RELEXER && !(settings.IGNORE_BOILERPLATE && settings.ThingsToIgnore.Contains(tok.Source)) {
-			println(tok.Type, tok.Literal)
-		}
 		if token.TokenTypeIsHeadword(tok.Type) {
 			if tok.Literal == "import" {
 				iz.Throw("init/import/first", &tok)

source/initializer/rsc-pf/worldlite.pf

@@ -8,8 +8,7 @@ Input = struct(prompt string)
 Output = struct()
 Terminal = struct()
 HTML = snippet
-
-Foo = enum FOO, BAR
+SQL = snippet
 
 cmd
 

source/lexer/lexer.go

@@ -64,7 +64,15 @@ func (l *Lexer) NextToken() token.Token {
 
 	switch l.ch {
 	case 0:
-		return l.NewToken(token.EOF, "EOF")
+		level := l.whitespaceStack.Find("")
+		if level > 0 {
+			for i := 0; i < level; i++ {
+				l.whitespaceStack.Pop()
+			}
+			return l.MakeToken(token.END, fmt.Sprint(level))
+		} else {
+			return l.NewToken(token.EOF, "EOF")
+		}
 	case '\n':
 		return l.NewToken(token.NEWLINE, ";")
 	case '\\':
@@ -198,7 +206,7 @@ func (l *Lexer) NextToken() token.Token {
 			l.readChar()
 			return l.NewToken(tType, text)
 		case token.EMDASH:
-			return l.MakeToken(tType, l.readSnippet())
+			return l.MakeToken(tType, strings.TrimSpace(l.readSnippet()))
 		default:
 			return l.NewToken(tType, lit)
 		}
@@ -393,8 +401,10 @@ func (l *Lexer) readSnippet() string {
 	}
 	// There are two possibilities. Either we found a non-whitespace character, and the whole snippet is on the same line as the
 	// `---` token, or we found a newline and the snipped is an indent on the succeeding lines. Just like with a colon.
-	if l.peekChar() == '\n' { // --- then we have to mess with whitespace.
-		l.readChar()
+	if l.peekChar() == '\n' || l.peekChar() == '\r' { // --- then we have to mess with whitespace.
+		for l.peekChar() == '\n' || l.peekChar() == '\r' {
+			l.readChar()
+		}
 		langIndent := ""
 		stackTop, ok := l.whitespaceStack.HeadValue()
 		if !ok {
@@ -417,8 +427,8 @@ func (l *Lexer) readSnippet() string {
 					return result
 				}
 			}
-			if strings.HasPrefix(stackTop, currentWhitespace) || currentWhitespace == "\n" { // Then we've unindented. Dobby is free!
-				if currentWhitespace != "\n" {
+			if strings.HasPrefix(stackTop, currentWhitespace) || currentWhitespace == "\n" || currentWhitespace == "\r" || currentWhitespace == string(0) { // Then we've unindented. Dobby is free!
+				if currentWhitespace != "\n" && currentWhitespace != "\r" && currentWhitespace == string(0) {
 					l.snippetWhitespace = currentWhitespace
 				}
 				return result
@@ -427,18 +437,19 @@ func (l *Lexer) readSnippet() string {
 				l.Throw("lex/emdash/indent/c", l.NewToken(token.ILLEGAL, "bad emdash"))
 				return result
 			}
-			for l.peekChar() != '\n' && l.peekChar() != 0 {
+			for l.peekChar() != '\n' && l.peekChar() != '\r' && l.peekChar() != 0 {
 				l.readChar()
 				result = result + string(l.ch)
 			}
 			if l.peekChar() == 0 {
+				l.readChar()
 				return result
 			}
 			l.readChar()
 			result = result + "\n"
 		}
 	} else {
-		for l.peekChar() != '\n' && l.peekChar() != 0 {
+		for l.peekChar() != '\n' && l.peekChar() != '\r' && l.peekChar() != 0 {
 			l.readChar()
 			result = result + string(l.ch)
 		}
@@ -621,7 +632,7 @@ func isHexDigit(ch rune) bool {
 
 func isProtectedPunctuationOrWhitespace(ch rune) bool {
 	return ch == '(' || ch == ')' || ch == '[' || ch == ']' || ch == '{' || ch == '}' || ch == ' ' || ch == ',' ||
-		ch == ':' || ch == ';' || ch == '\t' || ch == '\n' || ch == 0
+		ch == ':' || ch == ';' || ch == '\t' || ch == '\n' || ch == '\r' || ch == 0
 }
 
 func isSymbol(ch rune) bool {
@@ -646,7 +657,7 @@ func (l *Lexer) NewToken(tokenType token.TokenType, st string) token.Token {
 
 func (l *Lexer) MakeToken(tokenType token.TokenType, st string) token.Token {
 	if settings.SHOW_LEXER && !(settings.IGNORE_BOILERPLATE && settings.ThingsToIgnore.Contains(l.source)) {
-		fmt.Println(tokenType, st, l.line, l.tstart)
+		fmt.Println(tokenType, st)
 	}
 	return token.Token{Type: tokenType, Literal: st, Source: l.source, Line: l.line, ChStart: l.tstart, ChEnd: l.char}
 }

source/lexer/relexer.go

@@ -222,8 +222,12 @@ func (rl *Relexer) NextSemanticToken() token.Token {
 				return rl.burnToken()
 			}
 			rl.curTok.Literal = strconv.Itoa(n - 1)
-			return token.Token{Type: token.NEWLINE, Literal: ";", Line: rl.curTok.Line,
-				ChStart: 0, ChEnd: 0, Source: rl.curTok.Source}
+			if rl.nexTok.Type != token.EOF {
+				return token.Token{Type: token.NEWLINE, Literal: ";", Line: rl.curTok.Line,
+					ChStart: 0, ChEnd: 0, Source: rl.curTok.Source}
+			} else {
+				return rl.nexTok
+			}
 		default:
 			rl.nestingLevel = rl.nestingLevel - 1
 			rl.curTok.Literal = strconv.Itoa(n - 1)

source/parser/parser.go

@@ -10,6 +10,8 @@ import (
 	"github.com/tim-hardcastle/Pipefish/source/dtypes"
 	"github.com/tim-hardcastle/Pipefish/source/err"
 	"github.com/tim-hardcastle/Pipefish/source/lexer"
+	"github.com/tim-hardcastle/Pipefish/source/settings"
+	"github.com/tim-hardcastle/Pipefish/source/text"
 	"github.com/tim-hardcastle/Pipefish/source/token"
 	"github.com/tim-hardcastle/Pipefish/source/values"
 )
@@ -978,6 +980,9 @@ func (p *Parser) NextToken() {
 }
 
 func (p *Parser) SafeNextToken() {
+	if settings.SHOW_RELEXER && !(settings.IGNORE_BOILERPLATE && settings.ThingsToIgnore.Contains(p.curToken.Source)) {
+		println(text.PURPLE + p.curToken.Type, p.curToken.Literal + text.RESET)
+	}
 	p.curToken = p.peekToken
 	p.peekToken = p.TokenizedCode.NextToken()
 }

source/token/token.go

@@ -52,7 +52,7 @@ const (
 	LBRACE              = "{"
 	LBRACK              = "["
 	NAMESPACE_SEPARATOR = "."
-	NEWLINE             = "\n"
+	NEWLINE             = "NEWLINE"
 	RBRACE              = "}"
 	RBRACK              = "]"
 	RPAREN              = ")"