diff --git a/LineNumbers.md b/LineNumbers.md new file mode 100644 index 0000000..cd8be59 --- /dev/null +++ b/LineNumbers.md @@ -0,0 +1,324 @@ +# Linenumber directives + +Most languages have linenumber directives. They are used for generated code so +debuggers and errormessages are able to tell which line from the source file is +responsible instead of the line from the intermediate representation. Golang +uses '//line [filename]:[linenumber]', where [XXX] denotes an optional +parameter. Many (most?) languages from the C-family instead used the format +'#line lineno ["filename"]'. + +This is quite usefule, since `go build`, `go run` etc will provide references +to the markdown file with the bug (well, at least in theory). IF we want to +debug the intermediate file, a reference above the specific code is always +readily avialible a quick right click (Acme) or `gF` away (vim). + +If we want to implement this for lmt, we need to change quite a few things, the +most obvious is the internal representation of a line, which needs quite a lot +of more metadata, as the filename, the linenumber and the language of the +codeblock. + +We remake CodeBlock to a slice of CodeLines. For every line we don't only +record the text but the name of the source file, the linenumber and the +language (for different line directives). + +```go "global block variables" +type File string +type CodeBlock []CodeLine +type BlockName string +type language string +<<>> + +var blocks map[BlockName]CodeBlock +var files map[File]CodeBlock +``` + +```go "Codeline type definition" +type CodeLine struct { + text string + file File + lang language + number int +} +``` + +Handling a block line is now not as simple as just joining two strings. We +start appending CodeLines into CodeBlocks. + +```go "Handle block line" +block = append(block, line) +``` + +This cascades. The CodeBlocks in the maps files and blocks can't be joined, but +has to be appended. We take a few moments to clean up the emptying of metadata +per line, i.e. if a block is ending we can unset inBlock. + +```go "Handle block ending" +inBlock = false +// Update the files map if it's a file. +if fname != "" { + if appending { + files[fname] = append(files[fname], block...) + } else { + files[fname] = block + } +} + +// Update the named block map if it's a named block. +if bname != "" { + if appending { + blocks[bname] = append(blocks[bname], block...) + } else { + blocks[bname] = block + } +} +``` + +To get this to work we need to start using CodeLine as line in our +implementation of the file processing. We also should to save the filename +directly when opening the file. To simplify we break out our variables from the +implementation of file processing. + +```go "process file implementation variables" +scanner := bufio.NewReader(r) +var err error + +var line CodeLine +line.file = File(inputfilename) + +var inBlock, appending bool +var bname BlockName +var fname File +var block CodeBlock +``` + +When processing our files we need to increment the file line counter for every +line read. + +```go "process file implementation" +<<>> +for { + line.text, err = scanner.ReadString('\n') + line.number++ + switch err { + case io.EOF: + return nil + case nil: + // Nothing special + default: + return err + } + <<>> +} +``` + +Sadly, there is no indicator from ioreader which file we are reading so we send +it in with ProcessFile. + +```go "ProcessFile Declaration" +// Updates the blocks and files map for the markdown read from r. +func ProcessFile(r io.Reader, inputfilename string) error { + <<>> +} +``` + +Of course we need to call ProcessFile with the filename too. + +```go "Open and process file" +f, err := os.Open(file) +if err != nil { + fmt.Fprintln(os.Stderr, "error: ", err) + continue +} + +if err := ProcessFile(f, file); err != nil { + fmt.Fprintln(os.Stderr, "error: ", err) +} +// Don't defer since we're in a loop, we don't want to wait until the function +// exits. +f.Close() +``` + +When testing if the line starts with '```' we need to look at line.text, +instead of directly at line as before. + +```go "Handle file line" +if inBlock { + if line.text == "```\n" { + <<>> + continue + } + <<>> + continue +} +<<>> +``` + +Handle nonblock line is actually longer than needed. We only need to check if +it is a block start now, since markdown headers aren't handled as code block +descriptors. Lets clean it up. + +```go "Handle nonblock line" +<<>> +``` + +We check if a block starts by reading the first few characters (if the line is +long enough to hold them). If we are in the start of a new block, we reset +block (the variable we save every new block in), set inBlock to true and then +check the headers. + +```go "Check block start" +if len(line.text) >= 3 && (line.text[0:3] == "```") { + inBlock = true + // We were outside of a block, so just blindly reset it. + block = make(CodeBlock, 0) + <<>> +} +``` + +We parse the header for a new variable: the language of the codeblock. + +```go "Check block header" +fname, bname, appending, line.lang = parseHeader(line.text) +``` + +Parseheader must actually extract the new value, this quickly becomes rather +noisy, since both declaration and implementation changes. Worst part is that +the regexp changes too. We still probably should try looking at naming the +variables in the regexp. + +```go "ParseHeader Declaration" +func parseHeader(line string) (File, BlockName, bool, language) { + line = strings.TrimSpace(line) + <<>> +} +``` + +```go "parseHeader implementation" +var matches []string +if matches = namedBlockRe.FindStringSubmatch(line); matches != nil { + return "", BlockName(matches[2]), (matches[3] == "+="), language(matches[1]) +} +if matches = fileBlockRe.FindStringSubmatch(line); matches != nil { + return File(matches[2]), "", (matches[3] == "+="), language(matches[1]) +} +return "", "", false, "" +``` + +```go "Namedblock Regex" +namedBlockRe = regexp.MustCompile("^`{3,}\\s?(\\w*)\\s*\"(.+)\"\\s*([+][=])?$") +``` + +```go "Fileblock Regex" +fileBlockRe = regexp.MustCompile("^`{3,}\\s?(\\w+)\\s+([\\w\\.\\-\\/]+)\\s*([+][=])?$") +``` + +The (actual) replacement in Replace also joins strings, we need to append to +ret instead. The returned codeblock from the map blocks are replaced with the +new prefix and "expanded" (...) before appended. + +```go "Lookup replacement and add to ret" +bname := BlockName(matches[2]) +if val, ok := blocks[bname]; ok { + ret = append(ret, val.Replace(prefix+matches[1])...) +} else { + fmt.Fprintf(os.Stderr, "Warning: Block named %s referenced but not defined.\n", bname) + ret = append(ret, v) +} +``` + +Even lines which aren't replaced need to be appended instead of joining lines. +While we are at it let's cleanup empty lines when passing through (they should +not have prefix). + +```go "Handle replace line" +matches := replaceRe.FindStringSubmatch(line) +if matches == nil { + if v.text != "\n" { + v.text = prefix + v.text + } + ret = append(ret, v) + continue +} +<<>> +``` + +Finally something we can replace with a simpler implementation. Instead of +reading line by line from c (a CodeBlock, earlier a long string with newlines), +we're now able to use range. The string `line` is still analyzed, "de-macro-ed" +and prefixed. + +```go "Replace codeblock implementation" +var line string +for _, v := range c { + line = v.text + <<>> +} +return +``` + +We need to use all the information we gathered, lets add a "Finalize"-function +which handles CodeLines and outputs the final strings with the line-directives +we been working for. + +```go "other functions" += +<<>> +``` + +Let us implement a method which acts upon a codeblock and returns a string +containing the ready to be used textual representation of the code (or other +files). We check if the filename is the same as the previous line, and if the +linenumber has been increased by exactly one. If any of those have changed it +is because the source for this line are not the same as the previous and we +interject a "line directive". We know of two variants of these go and the +common from the C family of languages. lastly we save state, so we have +something to compare to on the next line. + +```go "Finalize Declaration" + +// Finalize extract the textual lines from CodeBlocks and (if needed) prepend a +// notice about "unexpected" filename or line changes, which is extracted from +// the contained CodeLines. The result is a string with newlines ready to be +// pasted into a file. +func (c CodeBlock) Finalize() (ret string) { + var file File + var formatstring string + var linenumber int + for _, l := range c { + if linenumber+1 != l.number || file != l.file { + switch l.lang { + case "go", "golang": + formatstring = "//line %[2]v:%[1]v\n" + default: + formatstring = "#line %v \"%v\"\n" + } + ret += fmt.Sprintf(formatstring, l.number, l.file) + } + ret += l.text + linenumber = l.number + file = l.file + } + return +} +``` + +And finally, lets use our Finalize on the Replace-d codeblock, right before we +print it out in the file. + +```go "Output files" +for filename, codeblock := range files { + if dir := filepath.Dir(string(filename)); dir != "." { + if err := os.MkdirAll(dir, 0775); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + } + } + + f, err := os.Create(string(filename)) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + continue + } + fmt.Fprintf(f, "%s", codeblock.Replace("").Finalize()) + // We don't defer this so that it'll get closed before the loop finishes. + f.Close() +} +``` diff --git a/MarkupExpansion.md b/MarkupExpansion.md new file mode 100644 index 0000000..14f026b --- /dev/null +++ b/MarkupExpansion.md @@ -0,0 +1,229 @@ +# Expand Markup + +lmt has chosen to be very selective about the markup it handles. By selecting +\<\<\>> as a markup for insertion macros it has chosen well. +Sadly it is not perfect, but what is? A common pain point is using gofmt on +codeblocks and ending up with garbage. By expanding the macro markup with the +option to use //\>> we are now able to gofmt our codeblocks +before passing through gofmt and ending up with perfect (well...) go code which +`go fmt` loves and respects. + +A second pain point is the inability to embed ``` in codeblocks, at least in a +way which most or any other markdown interpreters accepts. lmt has chosen to +start a code block with three (or more) backticks (by first checking if the +first character in a line is a backtick, then if three of them are and last by +removing all backticks at the beginning of a line, but at least one), and +ending a codeblock with a line with exactly three backticks. The original +markdown specs does not have code fences at all, but expect the writer to +intend the code with at least four spaces, lmt does not handle this, and it is +not a common enough occurance to even consider for lmt. [GFM] does state "The +content of the code block consists of all subsequent lines, until a closing +code fence of the same type as the code block began with (backticks or tildes), +and with at least as many backticks or tildes as the opening code fence" and "A +fenced code block begins with a code fence, indented no more than three +spaces". [CommonMark] does seem to be originator for the GFM description. +[Pandoc] make a case for "These begin with a row of three or more tildes (~) +and end with a row of tildes that must be at least as long as the starting row" +(but accepts backticks all the same in the next section). And [Markdown Extra] +fills in with "Fenced code blocks are like Markdown’s regular code blocks, +except that they’re not indented and instead rely on start and end fence lines +to delimit the code block. The code block starts with a line containing three +or more tilde ~ characters, and ends with the first line with the same number +of tilde ~" (but accepts backticks in the next section). Finally +[MultiMarkdown] is the outlier "These code blocks should begin with 3 to 5 +backticks, an optional language specifier (if using a syntax highlighter), and +should end with the same number of backticks you started with". + +[GFM]: https://github.github.com/gfm/#fenced-code-blocks +[CommonMark]: https://spec.commonmark.org/0.29/#fenced-code-blocks +[Pandoc]: https://pandoc.org/MANUAL.html#fenced-code-blocks +[Markdown Extra]: https://michelf.ca/projects/php-markdown/extra/#fenced-code-blocks +[MultiMarkdown]: https://fletcher.github.io/MultiMarkdown-4/syntax + +It does seem to make sense to expand (or at least "correct") the handling of +codeblocks. It seems we can sum it up with: + +- the character option of tildes and/or backticks +- to accept intendation +- and if so how much +- the number of characters +- should the block end with the same number of characters or at least the same? + +In short this proposal says both characters, allow intendation, three or more +starting, and at least the same at the end. It does seem to break as few as +possible of the other markdown implementations and should not break lmt-users +current files, while still allowing for having code blocks in markdown +formatted documentation code blocks (Inception). it does not touch upon the +format of the "info string" following the starting code fence. It is my solemn +belief this should hurt nobody and help somebody. + +As of now, these are our Regex implementations. + +```go "Namedblock Regex" +namedBlockRe = regexp.MustCompile("^`{3,}\\s?(\\w*)\\s*\"(.+)\"\\s*([+][=])?$") +``` + +```go "Fileblock Regex" +fileBlockRe = regexp.MustCompile("^`{3,}\\s?(\\w+)\\s+([\\w\\.\\-\\/]+)\\s*([+][=])?$") +``` + +```go "Replace Regex" +replaceRe = regexp.MustCompile(`^([\s]*)<<<(.+)>>>[\s]*$`) +``` + +Lets replace them one by one with something which implements the wishes above. + +Replace seems easiest. It changes `PREFIX<<>>optionalspaces` to also +allowing `PREFIX//>>optionalspaces`, where capital letters are named +capturing groups. We have decided NOT to capture the selection of <<< or //< +and not to allow for ending with //>. + +```go "Replace Regex" +replaceRe = regexp.MustCompile(`^(?P\s*)(?:<<|//)<(?P.+)>>>\s*$`) +``` + +We have to introduce some way of recording how the codefence is introduced, and +how many characters is in it, to be able to compare it with a "probable" ending +code fence, ergo a fence has one type of character and a lenght. + +```go "global block variables" += +type codefence struct { + char string // This should probably be a rune for purity + count int +} +``` + +Named blocks seems to be a wee bet harder than replace, but it is only at first +sight. First we have a "fence" of three or more tildes OR backticks, followed +by an optional space, the (optional) "language", optional spaces, the "name" of +the block in quotes and it ends with an "append" which is exactly `+=` if it +exists. + +```go "Namedblock Regex" +namedBlockRe = regexp.MustCompile("^(?P`{3,}|~{3,})\\s?(?P\\w*)\\s*\"(?P.+)\"\\s*(?P[+][=])?$") +``` + +File blocks are not that different. The "language" is no longer optional, and +must be followed by at least a space, and the name (or "file" in our +implementation) does not allow for a lot of different characters, but it is +probably good enough. + +```go "Fileblock Regex" +fileBlockRe = regexp.MustCompile("^(?P`{3,}|~{3,})\\s?(?P\\w+)\\s+(?P[\\w\\.\\-\\/]+)\\s*(?P[+][=])?$") +``` + +The worst thing is that both of the last regexps changes the matching order, by +introducing a new capturing group. All of our implementations need to change to +accommodate for this, and if we EVER change this again, they need to be +rechecked. To end this horror we've introduced names on our capturing groups, +but they aren't free to extract, we have to write some code for this. + +First we change the parsing of the header to use a a map of string to string +and introduce a new value "fence", which we return as a "codefence", introduced +earlier. We start to use a new function namedMatchesfromRe, which should return +a map from the names of captured groups to the contents therein provided a +regular expression and a string to match. + +```go "ParseHeader Declaration" +func parseHeader(line string) (File, BlockName, bool, language, codefence) { + line = strings.TrimSpace(line) // remove indentation and trailing spaces + + // lets iterate over the regexps we have. + for _, re := range []*regexp.Regexp{namedBlockRe, fileBlockRe} { + if m := namedMatchesfromRe(re, line); m != nil { + var fence codefence + fence.char = m["fence"][0:1] + fence.count = len(m["fence"]) + return File(m["file"]), BlockName(m["name"]), (m["append"] == "+="), language(m["language"]), fence + } + } + + // An empty return value for unnamed or broken fences to codeblocks. + return "", "", false, "", codefence{} +} +``` + +Our function to extract named groups from a string with a regular expression +does need to return nil if the regexp does not match the provided string! +Luckily this is easy, we could return early whenever this happens. The rest +follows, we extract the names from the regex, and runs through the matches, +using the names as indexes for our return value and lastly remove the result +from the unnamed groups. + +```go "Extract named matches from regexps" + +// namedMatchesfromRe takes an regexp and a string to match and returns a map +// of named groups to the matches. If not matches are found it returns nil. +func namedMatchesfromRe(re *regexp.Regexp, toMatch string) (ret map[string]string) { + substrings := re.FindStringSubmatch(toMatch) + if substrings == nil { + return nil + } + + ret = make(map[string]string) + names := re.SubexpNames() + + for i, s := range substrings { + ret[names[i]] = s + } + // The names[0] and names[x] from unnamed string are an empty string. + // Instead of checking every names[x] we simply overwrite the last ret[""] + // and discard it at the end. + delete(ret, "") + return +} +``` + +Add this to the functions for lmt. + +```go "other functions" += +<<>> +``` + +Parseheader returns a new value which we put in the variable fence. + +```go "Check block header" +fname, bname, appending, line.lang, fence = parseHeader(line.text) +``` + +The new variable needs to be defined before we process the file. + +```go "process file implementation variables" += +var fence codefence +``` + +And finally we're ready to start looking for codeblock ends which contain +something else than just three backticks. The length of the (trimmed) line must +be of equal length or longer than number of characters in the fence. The line +should also only contain fence characters, which we check by replacing all +fence characters and check if all that is left is the empty string. To +summarize: if we're not in a codeblock we should handle the line as a nonblock +line, otherwise we should look for an end to the codeblock OR handle block +lines. + +Note: strings.ReplaceAll was not introduced until go 1.12, which isn't +generally available in repositories. + +```go "Handle file line" +if !inBlock { + <<>> + continue +} +if l := strings.TrimSpace(line.text); len(l) >= fence.count && strings.Replace(l, fence.char, "", -1) == "" { + <<>> + continue +} +<<>> +``` + +Lastly, we check block starts by looking at the three first characters, and +accept not only backticks but tildes as well. + +```go "Check block start" +if len(line.text) >= 3 && (line.text[0:3] == "```" || line.text[0:3] == "~~~") { + inBlock = true + // We were outside of a block, so just blindly reset it. + block = make(CodeBlock, 0) + <<>> +} +``` diff --git a/README.md b/README.md index 26cf3d0..ba5ae8b 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,10 @@ to take that into account. Our maps, with some types defined for good measure: ```go "global variables" +<<>> +``` + +```go "global block variables" type File string type CodeBlock string type BlockName string @@ -115,6 +119,10 @@ var files map[File]CodeBlock Our ProcessFile function: ```go "other functions" +<<>> +``` + +```go "ProcessFile Declaration" // Updates the blocks and files map for the markdown read from r. func ProcessFile(r io.Reader) error { <<>> @@ -360,6 +368,10 @@ block = "" Then we need to define our parseHeader function: ```go "other functions" += +<<>> +``` + +```go "ParseHeader Declaration" func parseHeader(line string) (File, BlockName, bool) { line = strings.TrimSpace(line) <<>> @@ -391,6 +403,10 @@ var namedBlockRe *regexp.Regexp ``` ```go "Initialize" += +<<>> +``` + +```go "Namedblock Regex" namedBlockRe = regexp.MustCompile("^([`]+\\s?)[\\w]+[\\s]+\"(.+)\"[\\s]*([+][=])?$") ``` @@ -417,6 +433,10 @@ var fileBlockRe *regexp.Regexp ``` ```go "Initialize" += +<<>> +``` + +```go "Fileblock Regex" fileBlockRe = regexp.MustCompile("^([`]+\\s?)[\\w]+[\\s]+([\\w\\.\\-\\/]+)[\\s]*([+][=])?$") ``` diff --git a/main.go b/main.go index 7e5b2b9..183d41f 100644 --- a/main.go +++ b/main.go @@ -1,38 +1,71 @@ +//line README.md:65 package main import ( - "bufio" +//line README.md:149 "fmt" - "io" "os" - "path/filepath" + "io" +//line README.md:212 + "bufio" +//line README.md:385 "regexp" +//line README.md:510 "strings" +//line SubdirectoryFiles.md:35 + "path/filepath" +//line README.md:69 ) +//line LineNumbers.md:25 type File string -type CodeBlock string +type CodeBlock []CodeLine type BlockName string +type language string +//line LineNumbers.md:36 +type CodeLine struct { + text string + file File + lang language + number int +} +//line LineNumbers.md:30 var blocks map[BlockName]CodeBlock var files map[File]CodeBlock +//line MarkupExpansion.md:90 +type codefence struct { + char string // This should probably be a rune for purity + count int +} +//line README.md:402 var namedBlockRe *regexp.Regexp +//line README.md:432 var fileBlockRe *regexp.Regexp +//line README.md:516 var replaceRe *regexp.Regexp +//line README.md:72 +//line LineNumbers.md:118 // Updates the blocks and files map for the markdown read from r. -func ProcessFile(r io.Reader) error { +func ProcessFile(r io.Reader, inputfilename string) error { +//line LineNumbers.md:82 scanner := bufio.NewReader(r) var err error - var line string + + var line CodeLine + line.file = File(inputfilename) var inBlock, appending bool var bname BlockName var fname File var block CodeBlock - +//line MarkupExpansion.md:192 + var fence codefence +//line LineNumbers.md:99 for { - line, err = scanner.ReadString('\n') + line.text, err = scanner.ReadString('\n') + line.number++ switch err { case io.EOF: return nil @@ -41,122 +74,178 @@ func ProcessFile(r io.Reader) error { default: return err } - if inBlock { - if line == "```\n" { - // Update the files map if it's a file. - if fname != "" { - if appending { - files[fname] += block - } else { - files[fname] = block - } - } - - // Update the named block map if it's a named block. - if bname != "" { - if appending { - blocks[bname] += block - } else { - blocks[bname] = block - } - } - - inBlock = false - appending = false - bname = "" - fname = "" - block = "" - continue - } else { - block += CodeBlock(line) +//line MarkupExpansion.md:208 + if !inBlock { +//line MarkupExpansion.md:223 + if len(line.text) >= 3 && (line.text[0:3] == "```" || line.text[0:3] == "~~~") { + inBlock = true + // We were outside of a block, so just blindly reset it. + block = make(CodeBlock, 0) +//line MarkupExpansion.md:186 + fname, bname, appending, line.lang, fence = parseHeader(line.text) +//line MarkupExpansion.md:228 } - } else { - if line == "" { - continue +//line MarkupExpansion.md:210 + continue + } + if l := strings.TrimSpace(line.text); len(l) >= fence.count && strings.Replace(l, fence.char, "", -1) == "" { +//line LineNumbers.md:56 + inBlock = false + // Update the files map if it's a file. + if fname != "" { + if appending { + files[fname] = append(files[fname], block...) + } else { + files[fname] = block + } } - switch line[0] { - case '`': - if len(line) >= 3 && line[0:3] == "```" { - inBlock = true - fname, bname, appending = parseHeader(line) - // We're outside of a block, so just blindly reset it. - block = "" + // Update the named block map if it's a named block. + if bname != "" { + if appending { + blocks[bname] = append(blocks[bname], block...) + } else { + blocks[bname] = block } - default: - inBlock = false - appending = false - bname = "" - fname = "" - block = "" } +//line MarkupExpansion.md:214 + continue } +//line LineNumbers.md:48 + block = append(block, line) +//line LineNumbers.md:111 } +//line LineNumbers.md:121 } -func parseHeader(line string) (File, BlockName, bool) { - line = strings.TrimSpace(line) - matches := namedBlockRe.FindStringSubmatch(line) - if matches != nil { - return "", BlockName(matches[2]), (matches[3] == "+=") - } - matches = fileBlockRe.FindStringSubmatch(line) - if matches != nil { - return File(matches[2]), "", (matches[3] == "+=") +//line MarkupExpansion.md:128 +func parseHeader(line string) (File, BlockName, bool, language, codefence) { + line = strings.TrimSpace(line) // remove indentation and trailing spaces + + // lets iterate over the regexps we have. + for _, re := range []*regexp.Regexp{namedBlockRe, fileBlockRe} { + if m := namedMatchesfromRe(re, line); m != nil { + var fence codefence + fence.char = m["fence"][0:1] + fence.count = len(m["fence"]) + return File(m["file"]), BlockName(m["name"]), (m["append"] == "+="), language(m["language"]), fence + } } - return "", "", false -} + // An empty return value for unnamed or broken fences to codeblocks. + return "", "", false, "", codefence{} +} +//line WhitespacePreservation.md:34 // Replace expands all macros in a CodeBlock and returns a CodeBlock with no // references to macros. func (c CodeBlock) Replace(prefix string) (ret CodeBlock) { - scanner := bufio.NewReader(strings.NewReader(string(c))) - - for { - line, err := scanner.ReadString('\n') - // ReadString will eventually return io.EOF and this will return. - if err != nil { - return - } +//line LineNumbers.md:251 + var line string + for _, v := range c { + line = v.text +//line LineNumbers.md:234 matches := replaceRe.FindStringSubmatch(line) if matches == nil { - ret += CodeBlock(prefix) + CodeBlock(line) + if v.text != "\n" { + v.text = prefix + v.text + } + ret = append(ret, v) continue } +//line LineNumbers.md:220 bname := BlockName(matches[2]) if val, ok := blocks[bname]; ok { - ret += val.Replace(prefix + matches[1]) + ret = append(ret, val.Replace(prefix+matches[1])...) } else { fmt.Fprintf(os.Stderr, "Warning: Block named %s referenced but not defined.\n", bname) - ret += CodeBlock(line) + ret = append(ret, v) } +//line LineNumbers.md:255 } return +//line WhitespacePreservation.md:38 } +//line LineNumbers.md:277 + +// Finalize extract the textual lines from CodeBlocks and (if needed) prepend a +// notice about "unexpected" filename or line changes, which is extracted from +// the contained CodeLines. The result is a string with newlines ready to be +// pasted into a file. +func (c CodeBlock) Finalize() (ret string) { + var file File + var formatstring string + var linenumber int + for _, l := range c { + if linenumber+1 != l.number || file != l.file { + switch l.lang { + case "go", "golang": + formatstring = "//line %[2]v:%[1]v\n" + default: + formatstring = "#line %v \"%v\"\n" + } + ret += fmt.Sprintf(formatstring, l.number, l.file) + } + ret += l.text + linenumber = l.number + file = l.file + } + return +} +//line MarkupExpansion.md:154 + +// namedMatchesfromRe takes an regexp and a string to match and returns a map +// of named groups to the matches. If not matches are found it returns nil. +func namedMatchesfromRe(re *regexp.Regexp, toMatch string) (ret map[string]string) { + substrings := re.FindStringSubmatch(toMatch) + if substrings == nil { + return nil + } + + ret = make(map[string]string) + names := re.SubexpNames() + + for i, s := range substrings { + ret[names[i]] = s + } + // The names[0] and names[x] from unnamed string are an empty string. + // Instead of checking every names[x] we simply overwrite the last ret[""] + // and discard it at the end. + delete(ret, "") + return +} +//line README.md:74 func main() { +//line README.md:157 // Initialize the maps blocks = make(map[BlockName]CodeBlock) files = make(map[File]CodeBlock) - namedBlockRe = regexp.MustCompile("^([`]+\\s?)[\\w]+[\\s]+\"(.+)\"[\\s]*([+][=])?$") - fileBlockRe = regexp.MustCompile("^([`]+\\s?)[\\w]+[\\s]+([\\w\\.\\-\\/]+)[\\s]*([+][=])?$") - replaceRe = regexp.MustCompile(`^([\s]*)<<<(.+)>>>[\s]*$`) +//line MarkupExpansion.md:103 + namedBlockRe = regexp.MustCompile("^(?P`{3,}|~{3,})\\s?(?P\\w*)\\s*\"(?P.+)\"\\s*(?P[+][=])?$") +//line MarkupExpansion.md:112 + fileBlockRe = regexp.MustCompile("^(?P`{3,}|~{3,})\\s?(?P\\w+)\\s+(?P[\\w\\.\\-\\/]+)\\s*(?P[+][=])?$") +//line MarkupExpansion.md:82 + replaceRe = regexp.MustCompile(`^(?P\s*)(?:<<|//)<(?P.+)>>>\s*$`) +//line README.md:136 // os.Args[0] is the command name, "lmt". We don't want to process it. for _, file := range os.Args[1:] { +//line LineNumbers.md:127 f, err := os.Open(file) if err != nil { fmt.Fprintln(os.Stderr, "error: ", err) continue } - if err := ProcessFile(f); err != nil { + if err := ProcessFile(f, file); err != nil { fmt.Fprintln(os.Stderr, "error: ", err) } // Don't defer since we're in a loop, we don't want to wait until the function // exits. f.Close() +//line README.md:140 } +//line LineNumbers.md:308 for filename, codeblock := range files { if dir := filepath.Dir(string(filename)); dir != "." { if err := os.MkdirAll(dir, 0775); err != nil { @@ -169,9 +258,9 @@ func main() { fmt.Fprintf(os.Stderr, "%v\n", err) continue } - fmt.Fprintf(f, "%s", codeblock.Replace("")) + fmt.Fprintf(f, "%s", codeblock.Replace("").Finalize()) // We don't defer this so that it'll get closed before the loop finishes. f.Close() - } +//line README.md:77 }