diff --git a/src/SIL.Machine/Corpora/IUsfmParserHandler.cs b/src/SIL.Machine/Corpora/IUsfmParserHandler.cs index 4d7682f5..82c6e2db 100644 --- a/src/SIL.Machine/Corpora/IUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/IUsfmParserHandler.cs @@ -69,24 +69,24 @@ IReadOnlyList attributes void EndChar(UsfmParserState state, string marker, IReadOnlyList attributes, bool closed); /// - /// Start of a note + /// Start of a sub component - a note, figure or cross reference /// - void StartNote(UsfmParserState state, string marker, string caller, string category); + void StartSubComponent(UsfmParserState state, string marker, string caller, string category); /// - /// End of a note + /// End of a sub component /// - void EndNote(UsfmParserState state, string marker, bool closed); + void EndSubComponent(UsfmParserState state, string marker, bool closed); /// - /// Start of a note text + /// Start of a sub component text /// - void StartNoteText(UsfmParserState state); + void StartSubComponentText(UsfmParserState state); /// - /// End of a note text + /// End of a sub component text /// - void EndNoteText(UsfmParserState state); + void EndSubComponentText(UsfmParserState state); /// /// Start of a table diff --git a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs index 7441f94d..4a76a795 100644 --- a/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs +++ b/src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs @@ -24,7 +24,7 @@ public string UpdateUsfm( IReadOnlyList<(IReadOnlyList, string)> rows, string fullName = null, UpdateUsfmTextBehavior textBehavior = UpdateUsfmTextBehavior.PreferExisting, - UpdateUsfmIntraVerseMarkerBehavior noteBehavior = UpdateUsfmIntraVerseMarkerBehavior.Preserve, + UpdateUsfmIntraVerseMarkerBehavior subComponentBehavior = UpdateUsfmIntraVerseMarkerBehavior.Preserve, UpdateUsfmIntraVerseMarkerBehavior formattingBehavior = UpdateUsfmIntraVerseMarkerBehavior.Strip ) { @@ -42,7 +42,7 @@ public string UpdateUsfm( rows, fullName is null ? null : $"- {fullName}", textBehavior, - noteBehavior, + subComponentBehavior, formattingBehavior ); try diff --git a/src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs b/src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs index ca3c4e57..672577db 100644 --- a/src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs +++ b/src/SIL.Machine/Corpora/ScriptureRefUsfmParserHandlerBase.cs @@ -9,8 +9,8 @@ public enum ScriptureTextType None, NonVerse, Verse, - Note, - NoteText + SubComponent, + SubComponentText } public abstract class ScriptureRefUsfmParserHandlerBase : UsfmParserHandlerBase @@ -152,7 +152,7 @@ public override void EndSidebar(UsfmParserState state, string marker, bool close EndParentElement(); } - public override void StartNote(UsfmParserState state, string marker, string caller, string category) + public override void StartSubComponent(UsfmParserState state, string marker, string caller, string category) { if (CurrentTextType != ScriptureTextType.None && !_duplicateVerse) { @@ -162,15 +162,15 @@ public override void StartNote(UsfmParserState state, string marker, string call } } - public override void StartNoteText(UsfmParserState state) + public override void StartSubComponentText(UsfmParserState state) { - _curTextType.Push(ScriptureTextType.NoteText); - StartNoteText(state, CreateNonVerseRef()); + _curTextType.Push(ScriptureTextType.SubComponentText); + StartSubComponentText(state, CreateNonVerseRef()); } - public override void EndNoteText(UsfmParserState state) + public override void EndSubComponentText(UsfmParserState state) { - EndNoteText(state, CreateNonVerseRef()); + EndSubComponentText(state, CreateNonVerseRef()); _curTextType.Pop(); } @@ -206,9 +206,9 @@ protected virtual void StartNonVerseText(UsfmParserState state, ScriptureRef scr protected virtual void EndNonVerseText(UsfmParserState state, ScriptureRef scriptureRef) { } - protected virtual void StartNoteText(UsfmParserState state, ScriptureRef scriptureRef) { } + protected virtual void StartSubComponentText(UsfmParserState state, ScriptureRef scriptureRef) { } - protected virtual void EndNoteText(UsfmParserState state, ScriptureRef scriptureRef) { } + protected virtual void EndSubComponentText(UsfmParserState state, ScriptureRef scriptureRef) { } private void StartVerseText(UsfmParserState state) { diff --git a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs index 0fe073ea..54b2be2b 100644 --- a/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs +++ b/src/SIL.Machine/Corpora/UpdateUsfmParserHandler.cs @@ -28,7 +28,7 @@ public class UpdateUsfmParserHandler : ScriptureRefUsfmParserHandlerBase private readonly List _newTokens; private readonly string _idText; private readonly UpdateUsfmTextBehavior _textBehavior; - private readonly UpdateUsfmIntraVerseMarkerBehavior _noteBehavior; + private readonly UpdateUsfmIntraVerseMarkerBehavior _subComponentBehavior; private readonly UpdateUsfmIntraVerseMarkerBehavior _formattingBehavior; private readonly Stack _replace; private int _rowIndex; @@ -38,7 +38,7 @@ public UpdateUsfmParserHandler( IReadOnlyList<(IReadOnlyList, string)> rows = null, string idText = null, UpdateUsfmTextBehavior textBehavior = UpdateUsfmTextBehavior.PreferExisting, - UpdateUsfmIntraVerseMarkerBehavior noteBehavior = UpdateUsfmIntraVerseMarkerBehavior.Preserve, + UpdateUsfmIntraVerseMarkerBehavior subComponentBehavior = UpdateUsfmIntraVerseMarkerBehavior.Preserve, UpdateUsfmIntraVerseMarkerBehavior formattingBehavior = UpdateUsfmIntraVerseMarkerBehavior.Strip ) { @@ -48,7 +48,7 @@ public UpdateUsfmParserHandler( _idText = idText; _replace = new Stack(); _textBehavior = textBehavior; - _noteBehavior = noteBehavior; + _subComponentBehavior = subComponentBehavior; _formattingBehavior = formattingBehavior; } @@ -196,7 +196,7 @@ bool closed base.EndChar(state, marker, attributes, closed); } - public override void StartNote(UsfmParserState state, string marker, string caller, string category) + public override void StartSubComponent(UsfmParserState state, string marker, string caller, string category) { // strip out notes in verses that are being replaced if (ReplaceWithNewTokens(state)) @@ -204,18 +204,18 @@ public override void StartNote(UsfmParserState state, string marker, string call else CollectTokens(state); - base.StartNote(state, marker, caller, category); + base.StartSubComponent(state, marker, caller, category); } - public override void EndNote(UsfmParserState state, string marker, bool closed) + public override void EndSubComponent(UsfmParserState state, string marker, bool closed) { // strip out notes in verses that are being replaced - if (ReplaceWithNewTokens(state, closed: closed, endNote: true)) + if (ReplaceWithNewTokens(state, closed: closed, endSubComponent: true)) SkipTokens(state); else CollectTokens(state); - base.EndNote(state, marker, closed); + base.EndSubComponent(state, marker, closed); } public override void Ref(UsfmParserState state, string marker, string display, string target) @@ -284,13 +284,13 @@ protected override void EndNonVerseText(UsfmParserState state, ScriptureRef scri PopNewTokens(); } - protected override void StartNoteText(UsfmParserState state, ScriptureRef scriptureRef) + protected override void StartSubComponentText(UsfmParserState state, ScriptureRef scriptureRef) { IReadOnlyList rowTexts = AdvanceRows(new[] { scriptureRef }); PushNewTokens(rowTexts.Select(t => new UsfmToken(t + " "))); } - protected override void EndNoteText(UsfmParserState state, ScriptureRef scriptureRef) + protected override void EndSubComponentText(UsfmParserState state, ScriptureRef scriptureRef) { PopNewTokens(); } @@ -360,13 +360,13 @@ private void SkipTokens(UsfmParserState state) _tokenIndex = state.Index + 1 + state.SpecialTokenCount; } - private bool ReplaceWithNewTokens(UsfmParserState state, bool closed = true, bool endNote = false) + private bool ReplaceWithNewTokens(UsfmParserState state, bool closed = true, bool endSubComponent = false) { bool stripExistingText = _textBehavior == UpdateUsfmTextBehavior.StripExisting; bool newText = _replace.Count > 0 && _replace.Peek(); - bool inNote = state.NoteTag != null || endNote; - bool inNoteText = CurrentTextType == ScriptureTextType.NoteText; - bool isNoteTag = + bool inSubComponent = state.SubComponentTag != null || endSubComponent; + bool inSubComponentText = CurrentTextType == ScriptureTextType.SubComponentText; + bool isSubComponentTag = state.Token.Marker != null && UsfmStylesheet.IsNoteOrCrossReferencePart(state.Token.Marker); bool isFormattingTag = state.Token.Marker != null && !UsfmStylesheet.IsNoteOrCrossReferencePart(state.Token.Marker); @@ -379,7 +379,11 @@ private bool ReplaceWithNewTokens(UsfmParserState state, bool closed = true, boo bool useNewTokens = stripExistingText || (newText && !existingText) - || (newText && _textBehavior == UpdateUsfmTextBehavior.PreferNew && (!inNote || inNoteText)); + || ( + newText + && _textBehavior == UpdateUsfmTextBehavior.PreferNew + && (!inSubComponent || inSubComponentText) + ); if (useNewTokens && _newTokens.Count > 0) _tokens.AddRange(_newTokens); @@ -387,12 +391,12 @@ private bool ReplaceWithNewTokens(UsfmParserState state, bool closed = true, boo // figure out when to skip the existing text bool withinNewText = _replace.Any(r => r); - if (withinNewText && inNote) + if (withinNewText && inSubComponent) { - if (_noteBehavior == UpdateUsfmIntraVerseMarkerBehavior.Strip) + if (_subComponentBehavior == UpdateUsfmIntraVerseMarkerBehavior.Strip) return true; - if (!inNoteText || isNoteTag) + if (!inSubComponentText || isSubComponentTag) return false; } diff --git a/src/SIL.Machine/Corpora/UsfmParser.cs b/src/SIL.Machine/Corpora/UsfmParser.cs index b06d30f8..301fe88f 100644 --- a/src/SIL.Machine/Corpora/UsfmParser.cs +++ b/src/SIL.Machine/Corpora/UsfmParser.cs @@ -245,16 +245,18 @@ public bool ProcessToken() if (paraTag != null && paraTag.TextType != UsfmTextType.VerseText && paraTag.TextType != 0) CloseAll(); else - CloseNote(); + CloseSubComponent(); break; - case UsfmTokenType.Note: - CloseNote(); + case UsfmTokenType.SubComponent: + CloseSubComponent(); break; case UsfmTokenType.End: // If end marker for an active note - if (State.Stack.Any(e => e.Type == UsfmElementType.Note && (e.Marker + "*" == token.Marker))) + if ( + State.Stack.Any(e => e.Type == UsfmElementType.SubComponent && (e.Marker + "*" == token.Marker)) + ) { - CloseNote(closed: true); + CloseSubComponent(closed: true); break; } @@ -493,8 +495,8 @@ public bool ProcessToken() if (IsNoteText(token)) { // Note text should be handled as a full segment - State.Push(new UsfmParserElement(UsfmElementType.NoteText, token.Marker)); - Handler?.StartNoteText(State); + State.Push(new UsfmParserElement(UsfmElementType.SubComponentText, token.Marker)); + Handler?.StartSubComponentText(State); } string actualMarker; @@ -519,7 +521,7 @@ public bool ProcessToken() token.Attributes ); break; - case UsfmTokenType.Note: + case UsfmTokenType.SubComponent: // Look for category string noteCategory = null; if ( @@ -534,9 +536,9 @@ public bool ProcessToken() State.SpecialTokenCount += 3; } - State.Push(new UsfmParserElement(UsfmElementType.Note, token.Marker)); + State.Push(new UsfmParserElement(UsfmElementType.SubComponent, token.Marker)); - Handler?.StartNote(State, token.Marker, token.Data, noteCategory); + Handler?.StartSubComponent(State, token.Marker, token.Data, noteCategory); break; case UsfmTokenType.Text: string text = token.Text; @@ -606,15 +608,15 @@ public void CloseAll() private UsfmTokenType DetermineUnknownTokenType() { // Unknown inside notes are character - if (State.Stack.Any(e => e.Type == UsfmElementType.Note)) + if (State.Stack.Any(e => e.Type == UsfmElementType.SubComponent)) return UsfmTokenType.Character; return UsfmTokenType.Paragraph; } - private void CloseNote(bool closed = false) + private void CloseSubComponent(bool closed = false) { - if (State.Stack.Any(elem => elem.Type == UsfmElementType.Note)) + if (State.Stack.Any(elem => elem.Type == UsfmElementType.SubComponent)) { UsfmParserElement elem; do @@ -623,14 +625,14 @@ private void CloseNote(bool closed = false) break; elem = State.Peek(); - CloseElement(closed && elem.Type == UsfmElementType.Note); - } while (elem.Type != UsfmElementType.Note); + CloseElement(closed && elem.Type == UsfmElementType.SubComponent); + } while (elem.Type != UsfmElementType.SubComponent); } } private void CloseNoteText() { - while (State.Stack.Count > 0 && State.Peek().Type == UsfmElementType.NoteText) + while (State.Stack.Count > 0 && State.Peek().Type == UsfmElementType.SubComponentText) CloseElement(); } @@ -654,11 +656,11 @@ private void CloseElement(bool closed = false) case UsfmElementType.Char: Handler?.EndChar(State, element.Marker, element.Attributes, closed); break; - case UsfmElementType.Note: - Handler?.EndNote(State, element.Marker, closed); + case UsfmElementType.SubComponent: + Handler?.EndSubComponent(State, element.Marker, closed); break; - case UsfmElementType.NoteText: - Handler?.EndNoteText(State); + case UsfmElementType.SubComponentText: + Handler?.EndSubComponentText(State); break; case UsfmElementType.Table: Handler?.EndTable(State); @@ -694,7 +696,7 @@ private bool IsRef(UsfmToken token) private bool IsNoteText(UsfmToken token) { - return token.Marker == "ft" && State.Stack.Any(elem => elem.Type == UsfmElementType.Note); + return token.Marker == "ft" && State.Stack.Any(elem => elem.Type == UsfmElementType.SubComponent); } } } diff --git a/src/SIL.Machine/Corpora/UsfmParserHandlerBase.cs b/src/SIL.Machine/Corpora/UsfmParserHandlerBase.cs index 05609eb6..b9f8129f 100644 --- a/src/SIL.Machine/Corpora/UsfmParserHandlerBase.cs +++ b/src/SIL.Machine/Corpora/UsfmParserHandlerBase.cs @@ -53,13 +53,13 @@ public virtual void EndChar( bool closed ) { } - public virtual void StartNote(UsfmParserState state, string marker, string caller, string category) { } + public virtual void StartSubComponent(UsfmParserState state, string marker, string caller, string category) { } - public virtual void EndNote(UsfmParserState state, string marker, bool closed) { } + public virtual void EndSubComponent(UsfmParserState state, string marker, bool closed) { } - public virtual void StartNoteText(UsfmParserState state) { } + public virtual void StartSubComponentText(UsfmParserState state) { } - public virtual void EndNoteText(UsfmParserState state) { } + public virtual void EndSubComponentText(UsfmParserState state) { } public virtual void StartTable(UsfmParserState state) { } diff --git a/src/SIL.Machine/Corpora/UsfmParserState.cs b/src/SIL.Machine/Corpora/UsfmParserState.cs index 887b8c3b..0f78dd7c 100644 --- a/src/SIL.Machine/Corpora/UsfmParserState.cs +++ b/src/SIL.Machine/Corpora/UsfmParserState.cs @@ -105,13 +105,13 @@ public UsfmTag CharTag } /// - /// Current note tag or null for none + /// Current sub component tag (note, figure, cross reference) or null for none /// - public UsfmTag NoteTag + public UsfmTag SubComponentTag { get { - UsfmParserElement elem = Stack.LastOrDefault(e => e.Type == UsfmElementType.Note); + UsfmParserElement elem = Stack.LastOrDefault(e => e.Type == UsfmElementType.SubComponent); return elem != null ? Stylesheet.GetTag(elem.Marker) : null; } } @@ -153,7 +153,7 @@ public bool IsVerseText get { // Sidebars and notes are not verse text - if (_stack.Any(e => e.Type == UsfmElementType.Sidebar || e.Type == UsfmElementType.Note)) + if (_stack.Any(e => e.Type == UsfmElementType.Sidebar || e.Type == UsfmElementType.SubComponent)) return false; if (!IsVersePara) @@ -205,8 +205,8 @@ public enum UsfmElementType Table, Row, Cell, - Note, - NoteText, + SubComponent, + SubComponentText, Sidebar }; diff --git a/src/SIL.Machine/Corpora/UsfmStylesheet.cs b/src/SIL.Machine/Corpora/UsfmStylesheet.cs index a08ab3f8..641ca365 100644 --- a/src/SIL.Machine/Corpora/UsfmStylesheet.cs +++ b/src/SIL.Machine/Corpora/UsfmStylesheet.cs @@ -14,6 +14,9 @@ public class UsfmStylesheet private static readonly HashSet NoteTextTags = new HashSet { "ft", }; + private static readonly HashSet SubComponentTags = new HashSet { "fig", "rq", "f", "fe", "x", }; + + private static readonly HashSet UntranslatedParagraphTag = new HashSet { "r", "rem", }; private static readonly HashSet NoteAndCrossReferencePartTags = new HashSet { "f", @@ -150,9 +153,14 @@ public static bool IsNoteOrCrossReferencePart(string tag) return !(tag is null) && NoteAndCrossReferencePartTags.Contains(NonAlpha.Replace(tag ?? "", "")); } - public static bool IsFigure(string tag) + public static bool IsSubComponent(string tag) + { + return !(tag is null) && SubComponentTags.Contains(NonAlpha.Replace(tag ?? "", "")); + } + + public static bool IsUntranslatedParagraph(string tag) { - return tag == "fig"; + return !(tag is null) && UntranslatedParagraphTag.Contains(tag); } public static bool IsNoteText(string tag) diff --git a/src/SIL.Machine/Corpora/UsfmTextBase.cs b/src/SIL.Machine/Corpora/UsfmTextBase.cs index af2820e4..b3e9e948 100644 --- a/src/SIL.Machine/Corpora/UsfmTextBase.cs +++ b/src/SIL.Machine/Corpora/UsfmTextBase.cs @@ -199,16 +199,16 @@ bool closed _rowTexts.Peek().TrimEnd(); } - public override void StartNote(UsfmParserState state, string marker, string caller, string category) + public override void StartSubComponent(UsfmParserState state, string marker, string caller, string category) { - base.StartNote(state, marker, caller, category); + base.StartSubComponent(state, marker, caller, category); OutputMarker(state); } - public override void EndNote(UsfmParserState state, string marker, bool closed) + public override void EndSubComponent(UsfmParserState state, string marker, bool closed) { - base.EndNote(state, marker, closed); + base.EndSubComponent(state, marker, closed); if (closed) OutputMarker(state); @@ -293,7 +293,7 @@ protected override void EndNonVerseText(UsfmParserState state, ScriptureRef scri _rows.Add(_text.CreateRow(scriptureRef, text, _sentenceStart)); } - protected override void StartNoteText(UsfmParserState state, ScriptureRef scriptureRef) + protected override void StartSubComponentText(UsfmParserState state, ScriptureRef scriptureRef) { if (_text._includeMarkers) return; @@ -301,7 +301,7 @@ protected override void StartNoteText(UsfmParserState state, ScriptureRef script _rowTexts.Push(new StringBuilder()); } - protected override void EndNoteText(UsfmParserState state, ScriptureRef scriptureRef) + protected override void EndSubComponentText(UsfmParserState state, ScriptureRef scriptureRef) { if (_text._includeMarkers) return; diff --git a/src/SIL.Machine/Corpora/UsfmToken.cs b/src/SIL.Machine/Corpora/UsfmToken.cs index c0b105b9..5e07b134 100644 --- a/src/SIL.Machine/Corpora/UsfmToken.cs +++ b/src/SIL.Machine/Corpora/UsfmToken.cs @@ -13,7 +13,7 @@ public enum UsfmTokenType Text, Paragraph, Character, - Note, + SubComponent, End, Milestone, MilestoneEnd, diff --git a/src/SIL.Machine/Corpora/UsfmTokenizer.cs b/src/SIL.Machine/Corpora/UsfmTokenizer.cs index c03b836d..607cef54 100644 --- a/src/SIL.Machine/Corpora/UsfmTokenizer.cs +++ b/src/SIL.Machine/Corpora/UsfmTokenizer.cs @@ -183,6 +183,22 @@ public IReadOnlyList Tokenize(string usfm, bool preserveWhitespace = } ); } + else if (UsfmStylesheet.IsSubComponent(tag.Marker)) + { + tokens.Add( + new UsfmToken( + UsfmTokenType.SubComponent, + marker, + null, + endMarker, + GetNextWord(usfm, ref index, preserveWhitespace) + ) + { + LineNumber = lineNum, + ColumnNumber = colNum + } + ); + } else { tokens.Add( @@ -243,7 +259,7 @@ public IReadOnlyList Tokenize(string usfm, bool preserveWhitespace = case UsfmStyleType.Note: tokens.Add( new UsfmToken( - UsfmTokenType.Note, + UsfmTokenType.SubComponent, marker, null, endMarker, diff --git a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs index b7b4948f..d0d677f7 100644 --- a/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UpdateUsfmParserHandlerTests.cs @@ -76,7 +76,7 @@ public void GetUsfm_Verse_StripNote() (ScrRef("MAT 2:1"), "First verse of the second chapter.") }; - string target = UpdateUsfm(rows, noteBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip); + string target = UpdateUsfm(rows, subComponentBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip); Assert.That(target, Contains.Substring("\\v 1 First verse of the second chapter.\r\n")); } @@ -88,7 +88,7 @@ public void GetUsfm_Verse_StripNotesWithUpdatedVerseText() (ScrRef("MAT 1:1"), "First verse of the first chapter.") }; - string target = UpdateUsfm(rows, noteBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip); + string target = UpdateUsfm(rows, subComponentBehavior: UpdateUsfmIntraVerseMarkerBehavior.Strip); Assert.That(target, Contains.Substring("\\id MAT - Test\r\n")); Assert.That(target, Contains.Substring("\\ip An introduction to Matthew\\fe + \\ft This is an endnote.\\fe*")); Assert.That(target, Contains.Substring("\\v 1 First verse of the first chapter.\r\n\\li1\r\n\\v 2")); @@ -120,10 +120,10 @@ public void GetUsfm_Verse_PreserveFiguresAndReferences() // fig (ScrRef("MAT 1:5"), "Fifth verse of the first chapter."), (ScrRef("MAT 1:5/1:fig"), "figure text not updated"), + // r + (ScrRef("MAT 2:0/1:r"), "parallel reference not updated"), // rq (ScrRef("MAT 2:5/1:rq"), "quote reference not updated"), - // r - (ScrRef("MAT 2/1:r"), "parallel reference not updated"), // xo (ScrRef("MAT 2:6/3:xo"), "Cross reference not update"), // xt @@ -136,7 +136,7 @@ public void GetUsfm_Verse_PreserveFiguresAndReferences() Assert.That( target, Contains.Substring( - "\\v 1 First verse of the second chapter. \\f + \\fr 2:1: \\ft This is a new footnote.\\f*\r\n" + "\\v 5 Fifth verse of the first chapter.\r\n\\li2 \\fig Figure 1|src=\"image1.png\" size=\"col\" ref=\"1:5\"\\fig*\r\n\\v 6" ) ); } @@ -489,19 +489,25 @@ private static string UpdateUsfm( string? source = null, string? idText = null, UpdateUsfmTextBehavior textBehavior = UpdateUsfmTextBehavior.PreferNew, - UpdateUsfmIntraVerseMarkerBehavior noteBehavior = UpdateUsfmIntraVerseMarkerBehavior.Preserve, + UpdateUsfmIntraVerseMarkerBehavior subComponentBehavior = UpdateUsfmIntraVerseMarkerBehavior.Preserve, UpdateUsfmIntraVerseMarkerBehavior formattingBehavior = UpdateUsfmIntraVerseMarkerBehavior.Strip ) { if (source is null) { var updater = new FileParatextProjectTextUpdater(CorporaTestHelpers.UsfmTestProjectPath); - return updater.UpdateUsfm("MAT", rows, idText, textBehavior, noteBehavior, formattingBehavior); + return updater.UpdateUsfm("MAT", rows, idText, textBehavior, subComponentBehavior, formattingBehavior); } else { source = source.Trim().ReplaceLineEndings("\r\n") + "\r\n"; - var updater = new UpdateUsfmParserHandler(rows, idText, textBehavior, noteBehavior, formattingBehavior); + var updater = new UpdateUsfmParserHandler( + rows, + idText, + textBehavior, + subComponentBehavior, + formattingBehavior + ); UsfmParser.Parse(source, updater); return updater.GetUsfm(); } diff --git a/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs b/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs index c03db158..84a9c060 100644 --- a/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs +++ b/tests/SIL.Machine.Tests/Corpora/UsfmTokenizerTests.cs @@ -30,7 +30,7 @@ public void Tokenize() Assert.That(tokens[38].LineNumber, Is.EqualTo(11)); Assert.That(tokens[38].ColumnNumber, Is.EqualTo(1)); - Assert.That(tokens[47].Type, Is.EqualTo(UsfmTokenType.Note)); + Assert.That(tokens[47].Type, Is.EqualTo(UsfmTokenType.SubComponent)); Assert.That(tokens[47].Marker, Is.EqualTo("f")); Assert.That(tokens[47].Data, Is.EqualTo("+")); Assert.That(tokens[47].LineNumber, Is.EqualTo(11));