From 323382b6134260019f2c6d8996eb24321cf544bb Mon Sep 17 00:00:00 2001 From: Lee ByeongJun Date: Thu, 23 Jan 2025 23:45:49 +0900 Subject: [PATCH] chore: Remove fixer_v2 Lexer Files (#118) # Description Remove unnecessary files --- fixer_v2/query/hole.go | 73 +---------- fixer_v2/query/hole_test.go | 189 +-------------------------- fixer_v2/query/internal.go | 7 + fixer_v2/query/lexer.go | 111 ---------------- fixer_v2/query/meta_variable_test.go | 127 ------------------ fixer_v2/query/parser.go | 9 +- fixer_v2/query/parser_test.go | 18 +-- fixer_v2/query/type.go | 21 +++ 8 files changed, 44 insertions(+), 511 deletions(-) delete mode 100644 fixer_v2/query/lexer.go diff --git a/fixer_v2/query/hole.go b/fixer_v2/query/hole.go index f545076..a0cf5bd 100644 --- a/fixer_v2/query/hole.go +++ b/fixer_v2/query/hole.go @@ -132,80 +132,13 @@ func ParseHolePattern(pattern string) (*HoleConfig, error) { return config, nil } -func (l *Lexer) matchHole() bool { - if l.position+1 >= len(l.input) { - return false - } - startPos := l.position - - if l.input[l.position+1] == '[' { - isLongForm := (l.position+2 < len(l.input) && l.input[l.position+2] == '[') - end := l.findHoleEnd(isLongForm) - if end > 0 { - // Check for quantifier - if end < len(l.input) && isQuantifier(l.input[end]) { - end++ - } - - value := l.input[l.position:end] - config, err := ParseHolePattern(value) - if err != nil { - // If parsing fails, try to extract at least the name and create a basic config - basicName := extractHoleName(value) - basicConfig := HoleConfig{ - Name: basicName, - Type: HoleAny, - Quantifier: QuantNone, - } - l.addTokenWithHoleConfig(TokenHole, value, startPos, basicConfig) - } else { - // Create a token with the parsed configuration - l.addTokenWithHoleConfig(TokenHole, value, startPos, *config) - } - l.position = end - return true - } - } - return false -} - -func (l *Lexer) addTokenWithHoleConfig(tokenType TokenType, value string, pos int, config HoleConfig) { - l.tokens = append(l.tokens, Token{ - Type: tokenType, - Value: value, - Position: pos, - HoleConfig: &config, - }) +var quantifiers = map[byte]bool{ + '*': true, '+': true, '?': true, } // isQuantifier checks if a character is a valid quantifier func isQuantifier(c byte) bool { - return c == '*' || c == '+' || c == '?' -} - -func (l *Lexer) findHoleEnd(isLongForm bool) int { - if isLongForm { - for i := l.position + 3; i < len(l.input)-1; i++ { - if l.input[i] == ']' && l.input[i+1] == ']' { - // Check if there's a quantifier after the closing brackets - if i+2 < len(l.input) && isQuantifier(l.input[i+2]) { - return i + 3 - } - return i + 2 - } - } - } else { - for i := l.position + 2; i < len(l.input); i++ { - if l.input[i] == ']' { - // Check if there's a quantifier after the closing bracket - if i+1 < len(l.input) && isQuantifier(l.input[i+1]) { - return i + 2 - } - return i + 1 - } - } - } - return -1 + return quantifiers[c] } // extractHoleName extracts the hole name from a string like ":[name]" or ":[[name]]". diff --git a/fixer_v2/query/hole_test.go b/fixer_v2/query/hole_test.go index edc641b..123d195 100644 --- a/fixer_v2/query/hole_test.go +++ b/fixer_v2/query/hole_test.go @@ -71,194 +71,9 @@ func TestParseHolePattern(t *testing.T) { return } if err == nil { - if got.Name != tt.wantConfig.Name { - t.Errorf("Name = %v, want %v", got.Name, tt.wantConfig.Name) + if !got.Equal(*tt.wantConfig) { + t.Errorf("HoleConfig = %v, want %v", got, tt.wantConfig) } - if got.Type != tt.wantConfig.Type { - t.Errorf("Type = %v, want %v", got.Type, tt.wantConfig.Type) - } - if got.Quantifier != tt.wantConfig.Quantifier { - t.Errorf("Quantifier = %v, want %v", got.Quantifier, tt.wantConfig.Quantifier) - } - } - }) - } -} - -func TestMatchHoleWithConfig(t *testing.T) { - tests := []struct { - name string - input string - startPos int - wantMatch bool - wantToken Token - wantPos int - }{ - { - name: "basic hole", - input: ":[var] rest", - startPos: 0, - wantMatch: true, - wantToken: Token{ - Type: TokenHole, - Value: ":[var]", - Position: 0, - HoleConfig: &HoleConfig{ - Name: "var", - Type: HoleAny, - Quantifier: QuantNone, - }, - }, - wantPos: 6, - }, - { - name: "typed hole", - input: ":[[expr:expression]] rest", - startPos: 0, - wantMatch: true, - wantToken: Token{ - Type: TokenHole, - Value: ":[[expr:expression]]", - Position: 0, - HoleConfig: &HoleConfig{ - Name: "expr", - Type: HoleExpression, - Quantifier: QuantNone, - }, - }, - wantPos: 20, - }, - { - name: "hole with quantifier", - input: ":[[stmts:block]]* {", - startPos: 0, - wantMatch: true, - wantToken: Token{ - Type: TokenHole, - Value: ":[[stmts:block]]*", - Position: 0, - HoleConfig: &HoleConfig{ - Name: "stmts", - Type: HoleBlock, - Quantifier: QuantZeroOrMore, - }, - }, - wantPos: 17, - }, - { - name: "whitespace hole with optional", - input: ":[[ws:whitespace]]? rest", - startPos: 0, - wantMatch: true, - wantToken: Token{ - Type: TokenHole, - Value: ":[[ws:whitespace]]?", - Position: 0, - HoleConfig: &HoleConfig{ - Name: "ws", - Type: HoleWhitespace, - Quantifier: QuantZeroOrOne, - }, - }, - wantPos: 19, - }, - { - name: "identifier with one or more", - input: ":[[ids:identifier]]+ rest", - startPos: 0, - wantMatch: true, - wantToken: Token{ - Type: TokenHole, - Value: ":[[ids:identifier]]+", - Position: 0, - HoleConfig: &HoleConfig{ - Name: "ids", - Type: HoleIdentifier, - Quantifier: QuantOneOrMore, - }, - }, - wantPos: 20, - }, - { - name: "invalid hole format", - input: ":[invalid rest", - startPos: 0, - wantMatch: false, - wantPos: 0, - }, - { - name: "invalid type", - input: ":[[x:invalid]] rest", - startPos: 0, - wantMatch: true, // should match but create basic hole - wantToken: Token{ - Type: TokenHole, - Value: ":[[x:invalid]]", - Position: 0, - HoleConfig: &HoleConfig{ - Name: "x:invalid", - Type: HoleAny, - Quantifier: QuantNone, - }, - }, - wantPos: 14, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - l := NewLexer(tt.input) - l.position = tt.startPos - - got := l.matchHole() - if got != tt.wantMatch { - t.Errorf("matchHole() match = %v, want %v", got, tt.wantMatch) - return - } - - if !tt.wantMatch { - return - } - - if len(l.tokens) != 1 { - t.Errorf("matchHole() produced %d tokens, want 1", len(l.tokens)) - return - } - - gotToken := l.tokens[0] - - if gotToken.Type != tt.wantToken.Type { - t.Errorf("Token.Type = %v, want %v", gotToken.Type, tt.wantToken.Type) - } - if gotToken.Value != tt.wantToken.Value { - t.Errorf("Token.Value = %v, want %v", gotToken.Value, tt.wantToken.Value) - } - if gotToken.Position != tt.wantToken.Position { - t.Errorf("Token.Position = %v, want %v", gotToken.Position, tt.wantToken.Position) - } - - // Compare HoleConfig if present - if tt.wantToken.HoleConfig != nil { - if gotToken.HoleConfig == nil { - t.Errorf("Token.HoleConfig is nil, want %+v", tt.wantToken.HoleConfig) - } else { - if gotToken.HoleConfig.Name != tt.wantToken.HoleConfig.Name { - t.Errorf("HoleConfig.Name = %v, want %v", - gotToken.HoleConfig.Name, tt.wantToken.HoleConfig.Name) - } - if gotToken.HoleConfig.Type != tt.wantToken.HoleConfig.Type { - t.Errorf("HoleConfig.Type = %v, want %v", - gotToken.HoleConfig.Type, tt.wantToken.HoleConfig.Type) - } - if gotToken.HoleConfig.Quantifier != tt.wantToken.HoleConfig.Quantifier { - t.Errorf("HoleConfig.Quantifier = %v, want %v", - gotToken.HoleConfig.Quantifier, tt.wantToken.HoleConfig.Quantifier) - } - } - } - - if l.position != tt.wantPos { - t.Errorf("Lexer position = %v, want %v", l.position, tt.wantPos) } }) } diff --git a/fixer_v2/query/internal.go b/fixer_v2/query/internal.go index eb611ae..ba31ef4 100644 --- a/fixer_v2/query/internal.go +++ b/fixer_v2/query/internal.go @@ -1,5 +1,7 @@ package query +import "unicode" + /* State Transition Machine Design Rationale @@ -229,3 +231,8 @@ var identCharTable = [256]bool{ func isIdentChar(c byte) bool { return identCharTable[c] } + +// isWhitespace checks if the given byte is a space, tab, newline, etc. using unicode.IsSpace. +func isWhitespace(c byte) bool { + return unicode.IsSpace(rune(c)) +} diff --git a/fixer_v2/query/lexer.go b/fixer_v2/query/lexer.go deleted file mode 100644 index 2aecae4..0000000 --- a/fixer_v2/query/lexer.go +++ /dev/null @@ -1,111 +0,0 @@ -package query - -import ( - "unicode" -) - -//! TODO: remove this file. - -// Lexer is responsible for scanning the input string and producing tokens. -type Lexer struct { - input string // the entire input to tokenize - position int // current reading position in input - tokens []Token -} - -// NewLexer returns a new Lexer with the given input and initializes state. -func NewLexer(input string) *Lexer { - return &Lexer{ - input: input, - position: 0, - tokens: make([]Token, 0), - } -} - -// Tokenize processes the entire input and produces the list of tokens. -func (l *Lexer) Tokenize() []Token { - for l.position < len(l.input) { - currentPos := l.position - switch c := l.input[l.position]; { - // This might indicate a placeholder like :[name] or :[[name]] - case c == ':': - if l.matchHole() { - // If matchHole returns true, we found :[something] or :[[something]], - // and the position has been updated. Skip the default token creation. - continue - } - // If matchHole fails, we treat ':' as just a regular text token. - l.addToken(TokenText, string(c), currentPos) - l.position++ - - case c == '{': - l.addToken(TokenLBrace, "{", currentPos) - l.position++ - - case c == '}': - l.addToken(TokenRBrace, "}", currentPos) - l.position++ - - case isWhitespace(c): - l.lexWhitespace(currentPos) - l.position++ - - default: - // position incrementing is handled inside `lexText` - l.lexText(currentPos) - } - } - - // At the end, add an EOF token to indicate we're done. - l.addToken(TokenEOF, "", l.position) - return l.tokens -} - -// lexWhitespace scans consecutive whitespace and produces one TokenWhitespace. -func (l *Lexer) lexWhitespace(startPos int) { - start := l.position - for l.position < len(l.input) && isWhitespace(l.input[l.position]) { - l.position++ - } - // The substring from start..l.position is all whitespace - l.addToken(TokenWhitespace, l.input[start:l.position], startPos) - // Move back one so that the main loop can increment it again. - l.position-- -} - -// lexText scans consecutive non-special, non-whitespace characters to produce TokenText. -func (l *Lexer) lexText(startPos int) { - start := l.position - for l.position < len(l.input) { - c := l.input[l.position] - - // starting with `:[`? - if c == ':' && l.position+1 < len(l.input) && l.input[l.position+1] == '[' { - break - } - - if c == '{' || c == '}' || isWhitespace(c) { - break - } - - l.position++ - } - - if l.position > start { - l.addToken(TokenText, l.input[start:l.position], startPos) - } -} - -// addToken is a helper to append a new token to the lexer's token list. -func (l *Lexer) addToken(tokenType TokenType, value string, pos int) { - l.tokens = append(l.tokens, Token{ - Type: tokenType, - Value: value, - Position: pos, - }) -} - -// isWhitespace checks if the given byte is a space, tab, newline, etc. using unicode.IsSpace. -func isWhitespace(c byte) bool { - return unicode.IsSpace(rune(c)) -} diff --git a/fixer_v2/query/meta_variable_test.go b/fixer_v2/query/meta_variable_test.go index b1dd1fc..a08215a 100644 --- a/fixer_v2/query/meta_variable_test.go +++ b/fixer_v2/query/meta_variable_test.go @@ -4,133 +4,6 @@ import ( "testing" ) -func TestLexer(t *testing.T) { - tests := []struct { - name string - input string - expected []Token - }{ - { - name: "empty input", - input: "", - expected: []Token{ - {Type: TokenEOF, Value: "", Position: 0}, - }, - }, - { - name: "only whitespace", - input: " \t\n ", - expected: []Token{ - {Type: TokenWhitespace, Value: " \t\n ", Position: 0}, - {Type: TokenEOF, Value: "", Position: 7}, - }, - }, - { - name: "adjacent holes", - input: ":[a]:[b]:[[c]]", - expected: []Token{ - {Type: TokenHole, Value: ":[a]", Position: 0}, - {Type: TokenHole, Value: ":[b]", Position: 4}, - {Type: TokenHole, Value: ":[[c]]", Position: 8}, - {Type: TokenEOF, Value: "", Position: 14}, - }, - }, - { - name: "incomplete hole pattern", - input: ":[test", - expected: []Token{ - {Type: TokenText, Value: ":", Position: 0}, - {Type: TokenText, Value: "[test", Position: 1}, - {Type: TokenEOF, Value: "", Position: 6}, - }, - }, - { - name: "simple if condition", - input: "if :[[cond]] { return true }", - expected: []Token{ - {Type: TokenText, Value: "if", Position: 0}, - {Type: TokenWhitespace, Value: " ", Position: 2}, - {Type: TokenHole, Value: ":[[cond]]", Position: 3}, - {Type: TokenWhitespace, Value: " ", Position: 12}, - {Type: TokenLBrace, Value: "{", Position: 13}, - {Type: TokenWhitespace, Value: " ", Position: 14}, - {Type: TokenText, Value: "return", Position: 15}, - {Type: TokenWhitespace, Value: " ", Position: 21}, - {Type: TokenText, Value: "true", Position: 22}, - {Type: TokenWhitespace, Value: " ", Position: 26}, - {Type: TokenRBrace, Value: "}", Position: 27}, - {Type: TokenEOF, Value: "", Position: 28}, - }, - }, - { - name: "simple hole pattern", - input: ":[name]", - expected: []Token{ - {Type: TokenHole, Value: ":[name]", Position: 0}, - {Type: TokenEOF, Value: "", Position: 7}, - }, - }, - { - name: "double bracket hole pattern", - input: ":[[variable]]", - expected: []Token{ - {Type: TokenHole, Value: ":[[variable]]", Position: 0}, - {Type: TokenEOF, Value: "", Position: 13}, - }, - }, - { - name: "multiple holes", - input: "test :[a] :[b] :[[c]]", - expected: []Token{ - {Type: TokenText, Value: "test", Position: 0}, - {Type: TokenWhitespace, Value: " ", Position: 4}, - {Type: TokenHole, Value: ":[a]", Position: 5}, - {Type: TokenWhitespace, Value: " ", Position: 9}, - {Type: TokenHole, Value: ":[b]", Position: 10}, - {Type: TokenWhitespace, Value: " ", Position: 14}, - {Type: TokenHole, Value: ":[[c]]", Position: 15}, - {Type: TokenEOF, Value: "", Position: 21}, - }, - }, - { - name: "special characters between text", - input: "hello:{world}:test", - expected: []Token{ - {Type: TokenText, Value: "hello:", Position: 0}, - {Type: TokenLBrace, Value: "{", Position: 6}, - {Type: TokenText, Value: "world", Position: 7}, - {Type: TokenRBrace, Value: "}", Position: 12}, - {Type: TokenText, Value: ":", Position: 13}, - {Type: TokenText, Value: "test", Position: 14}, - {Type: TokenEOF, Value: "", Position: 18}, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - lexer := NewLexer(tt.input) - tokens := lexer.Tokenize() - - if len(tokens) != len(tt.expected) { - t.Errorf("Lexer.Tokenize() got %d tokens, want %d tokens", len(tokens), len(tt.expected)) - return - } - - for i, got := range tokens { - want := tt.expected[i] - if got.Type != want.Type || - got.Value != want.Value || - got.Position != want.Position { - t.Errorf("Token[%d] = {Type: %v, Value: %q, Position: %d}, want {Type: %v, Value: %q, Position: %d}", - i, got.Type, got.Value, got.Position, - want.Type, want.Value, want.Position) - } - } - }) - } -} - func TestHoleExtraction(t *testing.T) { tests := []struct { name string diff --git a/fixer_v2/query/parser.go b/fixer_v2/query/parser.go index a88551c..6bd4c6c 100644 --- a/fixer_v2/query/parser.go +++ b/fixer_v2/query/parser.go @@ -4,18 +4,19 @@ import ( "fmt" ) +// hole name -> position (optional usage) +type holes map[string]int + // Parser is supposed to consume tokens produced by the lexer and build an AST. type Parser struct { buffer *buffer current int tokens []Token - holes map[string]int // hole name -> position (optional usage) + holes holes } func NewParser() *Parser { - return &Parser{ - holes: make(map[string]int), - } + return &Parser{holes: make(holes)} } func (p *Parser) Parse(buf *buffer) ([]Node, error) { diff --git a/fixer_v2/query/parser_test.go b/fixer_v2/query/parser_test.go index 00c712f..b93548f 100644 --- a/fixer_v2/query/parser_test.go +++ b/fixer_v2/query/parser_test.go @@ -1,7 +1,6 @@ package query import ( - "reflect" "testing" ) @@ -80,7 +79,7 @@ func TestParser_ScanMetaVariable(t *testing.T) { return } - if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { + if !tt.wantErr && !got.Equal(tt.want) { t.Errorf("Parser.scanMetaVariable() = %v, want %v", got, tt.want) } }) @@ -137,7 +136,7 @@ func TestParser_ScanText(t *testing.T) { return } - if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { + if !tt.wantErr && !got.Equal(tt.want) { t.Errorf("Parser.scanText() = %v, want %v", got, tt.want) } }) @@ -184,13 +183,8 @@ func TestParser_ScanWhitespace(t *testing.T) { return } - if !tt.wantErr { - // Token의 주요 필드들만 비교 - if got.Type != tt.want.Type || - got.Value != tt.want.Value || - got.Position != tt.want.Position { - t.Errorf("Parser.scanWhitespace() = %v, want %v", got, tt.want) - } + if !tt.wantErr && !tt.want.Equal(got) { + t.Errorf("Parser.scanWhitespace() = %v, want %v", got, tt.want) } }) } @@ -236,7 +230,7 @@ func TestParser_ScanBrace(t *testing.T) { return } - if !tt.wantErr && !reflect.DeepEqual(got, tt.want) { + if !tt.wantErr && !tt.want.Equal(got) { t.Errorf("Parser.scanBrace() = %v, want %v", got, tt.want) } }) @@ -329,7 +323,7 @@ func TestParser_ParseTokenNode(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p := &Parser{ tokens: tt.tokens, - holes: make(map[string]int), + holes: make(holes), current: tt.current, } diff --git a/fixer_v2/query/type.go b/fixer_v2/query/type.go index 46ba83b..7a26b82 100644 --- a/fixer_v2/query/type.go +++ b/fixer_v2/query/type.go @@ -26,6 +26,19 @@ type Token struct { HoleConfig *HoleConfig // configuration for hole tokens (nil for non-hole tokens) } +func (t *Token) Equal(other Token) bool { + if t.Type != other.Type || t.Value != other.Value || t.Position != other.Position { + return false + } + if t.HoleConfig == nil && other.HoleConfig == nil { + return true + } + if t.HoleConfig == nil || other.HoleConfig == nil { + return false + } + return t.HoleConfig.Equal(*other.HoleConfig) +} + // NodeType defines different node types for AST construction. type NodeType int @@ -58,6 +71,7 @@ type PatternNode struct { } func (p *PatternNode) Type() NodeType { return NodePattern } + func (p *PatternNode) String() string { result := fmt.Sprintf("PatternNode(%d children):\n", len(p.Children)) for i, child := range p.Children { @@ -66,6 +80,7 @@ func (p *PatternNode) String() string { } return strings.TrimRight(result, "\n") } + func (p *PatternNode) Position() int { return p.pos } func (p *PatternNode) Equal(other Node) bool { _, ok := other.(*PatternNode) @@ -79,6 +94,12 @@ type HoleConfig struct { Name string } +func (h *HoleConfig) Equal(other HoleConfig) bool { + return h.Name == other.Name && + h.Type == other.Type && + h.Quantifier == other.Quantifier +} + // HoleNode represents a placeholder in the pattern like :[name] or :[[name]]. type HoleNode struct { Config HoleConfig