From de6c9c6d36c61df6e2a238d757c3c4aca6704a6e Mon Sep 17 00:00:00 2001 From: Wliu Date: Thu, 6 Apr 2017 20:15:15 -0400 Subject: [PATCH 1/5] Add 'alwaysMatchEndPattern' option to end patterns This option, when set to true, will force the end pattern to match, even when it is not the current rule. --- src/pattern.coffee | 72 ++++++++++++++++++++++++++++++---------------- src/rule.coffee | 21 ++++++++++---- src/scanner.coffee | 7 ++--- 3 files changed, 66 insertions(+), 34 deletions(-) diff --git a/src/pattern.coffee b/src/pattern.coffee index 3997992..f6241dd 100644 --- a/src/pattern.coffee +++ b/src/pattern.coffee @@ -8,7 +8,7 @@ module.exports = class Pattern constructor: (@grammar, @registry, options={}) -> {name, contentName, match, begin, end, patterns} = options - {captures, beginCaptures, endCaptures, applyEndPatternLast} = options + {captures, beginCaptures, endCaptures, applyEndPatternLast, @alwaysMatchEndPattern} = options {@include, @popRule, @hasBackReferences} = options @pushRule = null @@ -25,8 +25,8 @@ class Pattern else if begin @regexSource = begin @captures = beginCaptures ? captures - endPattern = @grammar.createPattern({match: end, captures: endCaptures ? captures, popRule: true}) - @pushRule = @grammar.createRule({@scopeName, @contentScopeName, patterns, endPattern, applyEndPatternLast}) + endPattern = @grammar.createPattern({match: end, captures: endCaptures ? captures, popRule: true, @alwaysMatchEndPattern}) + @pushRule = @grammar.createRule({@scopeName, @contentScopeName, patterns, endPattern, applyEndPatternLast, @alwaysMatchEndPattern}) if @captures? for group, capture of @captures @@ -92,7 +92,7 @@ class Pattern else "\\#{index}" - @grammar.createPattern({hasBackReferences: false, match: resolvedMatch, @captures, @popRule}) + @grammar.createPattern({hasBackReferences: false, match: resolvedMatch, @captures, @popRule, @alwaysMatchEndPattern}) ruleForInclude: (baseGrammar, name) -> hashIndex = name.indexOf('#') @@ -132,38 +132,60 @@ class Pattern else match - handleMatch: (stack, line, captureIndices, rule, endPatternMatch) -> + handleMatch: (stack, line, captureIndices, override) -> tags = [] zeroWidthMatch = captureIndices[0].start is captureIndices[0].end - if @popRule + if @popRule and override # Pushing and popping a rule based on zero width matches at the same index # leads to an infinite loop. We bail on parsing if we detect that case here. if zeroWidthMatch and _.last(stack).zeroWidthMatch and _.last(stack).rule.anchorPosition is captureIndices[0].end return false - {contentScopeName} = _.last(stack) - tags.push(@grammar.endIdForScope(contentScopeName)) if contentScopeName - else if @scopeName - scopeName = @resolveScopeName(@scopeName, line, captureIndices) - tags.push(@grammar.startIdForScope(scopeName)) + if @captures + tags.push(@tagsForCaptureIndices(line, _.clone(captureIndices), captureIndices, stack)...) + else + {start, end} = captureIndices[0] + tags.push(end - start) unless end is start - if @captures - tags.push(@tagsForCaptureIndices(line, _.clone(captureIndices), captureIndices, stack)...) - else - {start, end} = captureIndices[0] - tags.push(end - start) unless end is start - - if @pushRule - ruleToPush = @pushRule.getRuleToPush(line, captureIndices) - ruleToPush.anchorPosition = captureIndices[0].end - {contentScopeName} = ruleToPush - stack.push({rule: ruleToPush, scopeName, contentScopeName, zeroWidthMatch}) - tags.push(@grammar.startIdForScope(contentScopeName)) if contentScopeName + overrideTags = [] + while stack.length + {contentScopeName, scopeName, rule} = stack.pop() if @popRule + overrideTags.push(@grammar.endIdForScope(contentScopeName)) if contentScopeName + overrideTags.push(@grammar.endIdForScope(scopeName)) if scopeName + + break if rule.alwaysMatchEndPattern + + tags.unshift(overrideTags...) else - {scopeName} = stack.pop() if @popRule - tags.push(@grammar.endIdForScope(scopeName)) if scopeName + if @popRule + # Pushing and popping a rule based on zero width matches at the same index + # leads to an infinite loop. We bail on parsing if we detect that case here. + if zeroWidthMatch and _.last(stack).zeroWidthMatch and _.last(stack).rule.anchorPosition is captureIndices[0].end + return false + + {contentScopeName} = _.last(stack) + tags.push(@grammar.endIdForScope(contentScopeName)) if contentScopeName + else if @scopeName + scopeName = @resolveScopeName(@scopeName, line, captureIndices) + tags.push(@grammar.startIdForScope(scopeName)) + + if @captures + tags.push(@tagsForCaptureIndices(line, _.clone(captureIndices), captureIndices, stack)...) + else + {start, end} = captureIndices[0] + tags.push(end - start) unless end is start + + if @pushRule + ruleToPush = @pushRule.getRuleToPush(line, captureIndices) + ruleToPush.anchorPosition = captureIndices[0].end + {contentScopeName} = ruleToPush + stack.push({rule: ruleToPush, scopeName, contentScopeName, zeroWidthMatch}) + tags.push(@grammar.startIdForScope(contentScopeName)) if contentScopeName + else + {scopeName} = stack.pop() if @popRule + tags.push(@grammar.endIdForScope(scopeName)) if scopeName tags diff --git a/src/rule.coffee b/src/rule.coffee index 1ec2766..2982e5a 100644 --- a/src/rule.coffee +++ b/src/rule.coffee @@ -4,7 +4,7 @@ Scanner = require './scanner' module.exports = class Rule - constructor: (@grammar, @registry, {@scopeName, @contentScopeName, patterns, @endPattern, @applyEndPatternLast}={}) -> + constructor: (@grammar, @registry, {@scopeName, @contentScopeName, patterns, @endPattern, @applyEndPatternLast, @alwaysMatchEndPattern}={}) -> @patterns = [] for pattern in patterns ? [] @patterns.push(@grammar.createPattern(pattern)) unless pattern.disabled @@ -38,6 +38,13 @@ class Rule @scannersByBaseGrammarName[baseGrammar.name] = scanner scanner + getEndPatternScanner: (ruleStack) -> + patterns = @getIncludedPatterns(ruleStack.shift().rule.grammar) + for stack in ruleStack by -1 + patterns.unshift(stack.rule.endPattern) if stack.rule.endPattern?.alwaysMatchEndPattern + + new Scanner(patterns) + scanInjections: (ruleStack, line, position, firstLine) -> baseGrammar = ruleStack[0].rule.grammar if injections = baseGrammar.injections @@ -57,7 +64,11 @@ class Rule baseGrammar = ruleStack[0].rule.grammar results = [] - scanner = @getScanner(baseGrammar) + if @endPattern + scanner = @getEndPatternScanner(ruleStack.slice()) + else + scanner = @getScanner(baseGrammar) + if result = scanner.findNextMatch(lineWithNewline, firstLine, position, @anchorPosition) results.push(result) @@ -103,13 +114,13 @@ class Rule {index, captureIndices, scanner} = result [firstCapture] = captureIndices - endPatternMatch = @endPattern is scanner.patterns[index] - if nextTags = scanner.handleMatch(result, ruleStack, line, this, endPatternMatch) + override = @endPattern isnt scanner.patterns[index] and scanner.patterns[index].alwaysMatchEndPattern + if nextTags = scanner.handleMatch(result, ruleStack, line, override) {nextTags, tagsStart: firstCapture.start, tagsEnd: firstCapture.end} getRuleToPush: (line, beginPatternCaptureIndices) -> if @endPattern.hasBackReferences - rule = @grammar.createRule({@scopeName, @contentScopeName}) + rule = @grammar.createRule({@scopeName, @contentScopeName, @alwaysMatchEndPattern}) rule.endPattern = @endPattern.resolveBackReferences(line, beginPatternCaptureIndices) rule.patterns = [rule.endPattern, @patterns...] rule diff --git a/src/scanner.coffee b/src/scanner.coffee index 2d3660a..af34060 100644 --- a/src/scanner.coffee +++ b/src/scanner.coffee @@ -59,10 +59,9 @@ class Scanner # match - An object returned from a previous call to `findNextMatch`. # stack - An array of {Rule} objects. # line - The string being scanned. - # rule - The rule that matched. - # endPatternMatch - true if the rule's end pattern matched. + # override - true if an endPattern with `alwaysMatchEndPattern` matched. # # Returns an array of tokens representing the match. - handleMatch: (match, stack, line, rule, endPatternMatch) -> + handleMatch: (match, stack, line, override) -> pattern = @patterns[match.index] - pattern.handleMatch(stack, line, match.captureIndices, rule, endPatternMatch) + pattern.handleMatch(stack, line, match.captureIndices, override) From f0297b0597163289ca67fdb1d3231afbef3f25fc Mon Sep 17 00:00:00 2001 From: Wliu Date: Fri, 7 Apr 2017 22:54:54 -0400 Subject: [PATCH 2/5] Fix scope names not being popped correctly --- src/pattern.coffee | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pattern.coffee b/src/pattern.coffee index f6241dd..e6d68eb 100644 --- a/src/pattern.coffee +++ b/src/pattern.coffee @@ -153,9 +153,10 @@ class Pattern while stack.length {contentScopeName, scopeName, rule} = stack.pop() if @popRule overrideTags.push(@grammar.endIdForScope(contentScopeName)) if contentScopeName - overrideTags.push(@grammar.endIdForScope(scopeName)) if scopeName - break if rule.alwaysMatchEndPattern + break if rule.alwaysMatchEndPattern and rule.endPattern is this + + overrideTags.push(@grammar.endIdForScope(scopeName)) if scopeName tags.unshift(overrideTags...) else From 90e610f47e1678b17b20885ca0413ee0eb23b59d Mon Sep 17 00:00:00 2001 From: Wliu Date: Fri, 7 Apr 2017 22:54:59 -0400 Subject: [PATCH 3/5] Specs --- spec/fixtures/always-match-end-pattern.cson | 38 +++++++++++++++++++++ spec/grammar-spec.coffee | 29 ++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 spec/fixtures/always-match-end-pattern.cson diff --git a/spec/fixtures/always-match-end-pattern.cson b/spec/fixtures/always-match-end-pattern.cson new file mode 100644 index 0000000..d8436ae --- /dev/null +++ b/spec/fixtures/always-match-end-pattern.cson @@ -0,0 +1,38 @@ +name: "alwaysMatchEndPattern" +scopeName: "source.always-match-end-pattern" +patterns: [ + { + begin: 'outer-inner' + end: '```' + name: 'outer' + alwaysMatchEndPattern: true + patterns: [ + { + begin: '/\\*' + end: '\\*/' + name: 'inner' + } + ] + } + { + begin: 'outer-middle-inner' + end: '```' + name: 'outer' + alwaysMatchEndPattern: true + patterns: [ + { + begin: '/\\*' + end: '```' + name: 'middle' + alwaysMatchEndPattern: true + patterns: [ + { + begin: '' + name: 'inner' + } + ] + } + ] + } +] diff --git a/spec/grammar-spec.coffee b/spec/grammar-spec.coffee index 1a55239..af83cd9 100644 --- a/spec/grammar-spec.coffee +++ b/spec/grammar-spec.coffee @@ -295,6 +295,35 @@ describe "Grammar tokenization", -> expect(lines[4][2]).toEqual value: "}", scopes: ['source.apply-end-pattern-last', 'normal-env', 'scope'] expect(lines[4][3]).toEqual value: "excentricSyntax }", scopes: ['source.apply-end-pattern-last', 'normal-env'] + describe "when the alwaysMatchEndPattern flag is set in a pattern", -> + beforeEach -> + grammar = loadGrammarSync('always-match-end-pattern.cson') + + it "attempts to match that end pattern first even when its included patterns have not finished matching", -> + lines = grammar.tokenizeLines """ + outer-inner + /* + stuff + ``` + """ + + expect(lines[1][0]).toEqual value: '/*', scopes: ['source.always-match-end-pattern', 'outer', 'inner'] + expect(lines[3][0]).toEqual value: '```', scopes: ['source.always-match-end-pattern', 'outer'] + + it "attempts to match the outermost pattern", -> + lines = grammar.tokenizeLines """ + outer-middle-inner + /* + stuff +