Skip to content

Commit cb24403

Browse files
vschvsch
committed
fix tracked offset with preserve spaces at last non-blank position
1 parent 9032116 commit cb24403

11 files changed

Lines changed: 96 additions & 48 deletions

File tree

‎.idea/codeStyles/Project.xml‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎VERSION-TODO.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
- [Release 0.60.0](#release-0600)
88
- [API Refactoring](#api-refactoring)
99
- [Next 0.61.xx](#next-061xx)
10+
- [0.61.26](#06126)
1011
- [0.61.24](#06124)
1112
- [0.61.22](#06122)
1213
- [0.61.20](#06120)
@@ -224,6 +225,16 @@ Please give feedback on the upcoming changes if you have concerns about breaking
224225
* [ ] Fix: Html converter to not add spaces between end of inline marker and next punctuation:
225226
`.,:;`
226227

228+
## 0.61.26
229+
230+
* Fix: `SequenceBuilder.toString()` not to add space between sequence parts.
231+
* Break: `TextCollectingVisitor` needs `TextContainer.F_ADD_SPACES_BETWEEN_NODES` to ensure
232+
there is at least one space between texts of different nodes in collected text. Previously
233+
this was added automatically by sequence builder. Use one of the `getAndCollect` functions
234+
taking options flags.
235+
* Fix: `MarkdownParagraph` wrapping with preserving tracked offsets with spaces to preserve
236+
spaces right before last non-blank character.
237+
227238
## 0.61.24
228239

229240
* Fix: link refs in link text should be collapsed.

‎flexmark-core-test/src/test/resources/core_wrapping_spec.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1593,3 +1593,50 @@ BasedSegmentBuilder{[0, 104), s=0:0, u=1:1, t=1:1, l=105, sz=3, na=2: [0, 104),
15931593
````````````````````````````````
15941594

15951595

1596+
```````````````````````````````` example(Wrap - Restore Spaces: 54) options(margin[76], insert-space, restore-tracked-spaces, show-ranges, running-tests)
1597+
[B ⦙]
1598+
.
1599+
[B ⦙]
1600+
---- Tracked Offsets ---------------------------------------------------
1601+
[0]: {3 1|0 si -> 3}
1602+
1603+
---- Ranges ------------------------------------------------------------
1604+
⟦[B ]⟧
1605+
⟦⟧
1606+
---- Segments ----------------------------------------------------------
1607+
BasedSegmentBuilder{[0, 4), s=0:0, u=1:1, t=1:1, l=5, sz=3, na=2: [0, 4), a:'\n', [4) }
1608+
````````````````````````````````
1609+
1610+
1611+
```````````````````````````````` example(Wrap - Restore Spaces: 55) options(margin[76], show-ranges)
1612+
[B ]
1613+
.
1614+
[B]
1615+
---- Ranges ------------------------------------------------------------
1616+
⟦[B⟧⟦]⟧
1617+
⟦⟧
1618+
---- Segments ----------------------------------------------------------
1619+
BasedSegmentBuilder{[0, 4), s=0:0, u=1:1, t=1:1, l=4, sz=4, na=3: [0, 2), [3, 4), a:'\n', [4) }
1620+
.
1621+
Document[0, 5]
1622+
Paragraph[0, 5]
1623+
LinkRef[0, 4] referenceOpen:[0, 1, "["] reference:[1, 2, "B"] referenceClose:[3, 4, "]"]
1624+
Text[1, 2] chars:[1, 2, "B"]
1625+
````````````````````````````````
1626+
1627+
1628+
```````````````````````````````` example(Wrap - Restore Spaces: 56) options(margin[76], restore-tracked-spaces, show-ranges)
1629+
[B ⦙]
1630+
.
1631+
[B ⦙]
1632+
---- Tracked Offsets ---------------------------------------------------
1633+
[0]: {3 1|0 -> 3}
1634+
1635+
---- Ranges ------------------------------------------------------------
1636+
⟦[B ]⟧
1637+
⟦⟧
1638+
---- Segments ----------------------------------------------------------
1639+
BasedSegmentBuilder{[0, 4), s=0:0, u=1:1, t=1:1, l=5, sz=3, na=2: [0, 4), a:'\n', [4) }
1640+
````````````````````````````````
1641+
1642+

‎flexmark-ext-tables/src/test/java/com/vladsch/flexmark/ext/tables/TableTextCollectingVisitorTest.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ public void test_basic() {
3232
"";
3333
Node document = parser.parse(markdown);
3434
TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
35-
final String text = collectingVisitor.collectAndGetText(document);
35+
final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_ADD_SPACES_BETWEEN_NODES);
3636
//System.out.println(text);
3737

3838
//final String astText = new AstCollectingVisitor().collectAndGetAstText(document);
@@ -61,7 +61,7 @@ public void test_linkURL() {
6161
"";
6262
Node document = parser.parse(markdown);
6363
TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
64-
final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_URL);
64+
final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_URL | TextContainer.F_ADD_SPACES_BETWEEN_NODES);
6565
//System.out.println(text);
6666

6767
//final String astText = new AstCollectingVisitor().collectAndGetAstText(document);
@@ -86,7 +86,7 @@ public void test_linkNodeText() {
8686
"";
8787
Node document = parser.parse(markdown);
8888
TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
89-
final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_NODE_TEXT);
89+
final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_NODE_TEXT | TextContainer.F_ADD_SPACES_BETWEEN_NODES);
9090
//System.out.println(text);
9191

9292
//final String astText = new AstCollectingVisitor().collectAndGetAstText(document);
@@ -111,14 +111,14 @@ public void test_linkUrlNodeText() {
111111
"";
112112
Node document = parser.parse(markdown);
113113
TextCollectingVisitor collectingVisitor = new TextCollectingVisitor();
114-
final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_URL | TextContainer.F_NODE_TEXT);
114+
final String text = collectingVisitor.collectAndGetText(document, TextContainer.F_LINK_URL | TextContainer.F_ADD_SPACES_BETWEEN_NODES);
115115
//System.out.println(text);
116116

117117
//final String astText = new AstCollectingVisitor().collectAndGetAstText(document);
118118
//System.out.println(astText);
119119
assertEquals("" +
120120
"First Header Second Header\n" +
121-
"**Content Cell** ![](image%20spaces.png)\n" +
121+
"Content Cell image spaces.png\n" +
122122
"", text);
123123
}
124124
}

‎flexmark-util-ast/src/main/java/com/vladsch/flexmark/util/ast/SpaceInsertingSequenceBuilder.java‎

Lines changed: 20 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package com.vladsch.flexmark.util.ast;
22

3+
import com.vladsch.flexmark.util.misc.BitFieldSet;
34
import com.vladsch.flexmark.util.misc.CharPredicate;
45
import com.vladsch.flexmark.util.sequence.BasedSequence;
56
import com.vladsch.flexmark.util.sequence.Range;
@@ -13,36 +14,38 @@
1314
public class SpaceInsertingSequenceBuilder implements ISequenceBuilder<SpaceInsertingSequenceBuilder, BasedSequence> {
1415
@NotNull
1516
public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull BasedSequence base) {
16-
return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base));
17+
return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base), false);
1718
}
1819

1920
@NotNull
2021
public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull BasedSequence base, @NotNull SegmentOptimizer optimizer) {
21-
return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, optimizer));
22+
return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, optimizer), false);
2223
}
2324

2425
@NotNull
2526
public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull BasedSequence base, int options) {
26-
return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, options));
27+
return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, options), BitFieldSet.any(options, TextContainer.F_ADD_SPACES_BETWEEN_NODES));
2728
}
2829

2930
@NotNull
3031
public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull BasedSequence base, int options, @NotNull SegmentOptimizer optimizer) {
31-
return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, options, optimizer));
32+
return new SpaceInsertingSequenceBuilder(SequenceBuilder.emptyBuilder(base, options, optimizer), BitFieldSet.any(options, TextContainer.F_ADD_SPACES_BETWEEN_NODES));
3233
}
3334

3435
@NotNull
3536
public static SpaceInsertingSequenceBuilder emptyBuilder(@NotNull SequenceBuilder builder) {
36-
return new SpaceInsertingSequenceBuilder(builder);
37+
return new SpaceInsertingSequenceBuilder(builder, false);
3738
}
3839

3940
final SequenceBuilder out;
4041
Node lastNode;
41-
boolean addSpaceOnNonBlank;
4242
boolean needEol;
43+
final boolean addSpacesBetweenNodes;
44+
boolean addSpaces;
4345

44-
private SpaceInsertingSequenceBuilder(SequenceBuilder out) {
46+
private SpaceInsertingSequenceBuilder(SequenceBuilder out, boolean addSpacesBetweenNodes) {
4547
this.out = out;
48+
this.addSpacesBetweenNodes = addSpacesBetweenNodes;
4649
}
4750

4851
public SequenceBuilder getOut() {
@@ -54,14 +57,6 @@ public char charAt(int index) {
5457
return out.charAt(index);
5558
}
5659

57-
public boolean isAddSpaceOnNonBlank() {
58-
return addSpaceOnNonBlank;
59-
}
60-
61-
public void setAddSpaceOnNonBlank(boolean addSpaceOnNonBlank) {
62-
this.addSpaceOnNonBlank = addSpaceOnNonBlank;
63-
}
64-
6560
public boolean isNeedEol() {
6661
return needEol;
6762
}
@@ -79,10 +74,10 @@ public void setLastNode(Node lastNode) {
7974

8075
if (this.lastNode != null && this.lastNode.getEndOffset() < lastNode.getStartOffset()) {
8176
BasedSequence sequence = getBaseSequence().subSequence(this.lastNode.getEndOffset(), lastNode.getStartOffset());
82-
this.addSpaceOnNonBlank = sequence.indexOfAny(CharPredicate.SPACE_TAB_EOL) != -1;
8377
this.needEol = sequence.trim(CharPredicate.SPACE_TAB).length() > 0 && sequence.trim(CharPredicate.WHITESPACE).isEmpty();
8478
}
8579

80+
addSpaces = addSpacesBetweenNodes;
8681
this.lastNode = lastNode;
8782
}
8883

@@ -155,14 +150,14 @@ public boolean needEol() {
155150

156151
@Override
157152
@NotNull
158-
public SpaceInsertingSequenceBuilder getBuilder() {return new SpaceInsertingSequenceBuilder(out.getBuilder());}
153+
public SpaceInsertingSequenceBuilder getBuilder() {return new SpaceInsertingSequenceBuilder(out.getBuilder(), addSpacesBetweenNodes);}
159154

160155
@Override
161156
@NotNull
162157
public SpaceInsertingSequenceBuilder append(@Nullable CharSequence chars, int startIndex, int endIndex) {
163-
if (addSpaceOnNonBlank && chars != null && startIndex < endIndex && !CharPredicate.WHITESPACE.test(chars.charAt(startIndex)) && needSpace()) {
158+
if (addSpaces && chars != null && startIndex < endIndex && !CharPredicate.WHITESPACE.test(chars.charAt(startIndex)) && needSpace()) {
164159
out.append(' ');
165-
addSpaceOnNonBlank = false;
160+
addSpaces = false;
166161
}
167162
out.append(chars, startIndex, endIndex);
168163
return this;
@@ -171,9 +166,9 @@ public SpaceInsertingSequenceBuilder append(@Nullable CharSequence chars, int st
171166
@Override
172167
@NotNull
173168
public SpaceInsertingSequenceBuilder append(char c) {
174-
if (addSpaceOnNonBlank && !CharPredicate.WHITESPACE.test(c) && needSpace()) {
169+
if (addSpaces && !CharPredicate.WHITESPACE.test(c) && needSpace()) {
175170
out.append(' ');
176-
addSpaceOnNonBlank = false;
171+
addSpaces = false;
177172
}
178173
out.append(c);
179174
return this;
@@ -182,19 +177,19 @@ public SpaceInsertingSequenceBuilder append(char c) {
182177
@Override
183178
@NotNull
184179
public SpaceInsertingSequenceBuilder append(char c, int count) {
185-
if (addSpaceOnNonBlank && !CharPredicate.WHITESPACE.test(c) && needSpace()) {
180+
if (addSpaces && !CharPredicate.WHITESPACE.test(c) && needSpace()) {
186181
out.append(' ');
187-
addSpaceOnNonBlank = false;
182+
addSpaces = false;
188183
}
189184
out.append(c, count);
190185
return this;
191186
}
192187

193188
@NotNull
194189
public SpaceInsertingSequenceBuilder append(int startOffset, int endOffset) {
195-
if (addSpaceOnNonBlank && startOffset < endOffset && !CharPredicate.WHITESPACE.test(out.getBaseSequence().charAt(startOffset)) && needSpace()) {
190+
if (addSpaces && startOffset < endOffset && !CharPredicate.WHITESPACE.test(out.getBaseSequence().charAt(startOffset)) && needSpace()) {
196191
out.append(' ');
197-
addSpaceOnNonBlank = false;
192+
addSpaces = false;
198193
}
199194
out.append(startOffset, endOffset);
200195
return this;

‎flexmark-util-ast/src/main/java/com/vladsch/flexmark/util/ast/TextCollectingVisitor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ public BasedSequence collectAndGetSequence(Node node) {
6262
}
6363

6464
public void collect(Node node, int flags) {
65-
out = SpaceInsertingSequenceBuilder.emptyBuilder(node.getChars());
65+
out = SpaceInsertingSequenceBuilder.emptyBuilder(node.getChars(), flags);
6666
this.flags = flags;
6767
myVisitor.visit(node);
6868
}

‎flexmark-util-ast/src/main/java/com/vladsch/flexmark/util/ast/TextContainer.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ enum Flags implements BitField {
1212
FOR_HEADING_ID, // text for heading ID
1313
NO_TRIM_REF_TEXT_START, // don't trim ref text start
1414
NO_TRIM_REF_TEXT_END, // don't trim ref text end
15+
ADD_SPACES_BETWEEN_NODES, // when appending text from different nodes, ensure there is at least one space
1516
;
1617

1718
final int bits;
@@ -41,6 +42,7 @@ public int getBits() {
4142
int F_FOR_HEADING_ID = BitFieldSet.intMask(Flags.FOR_HEADING_ID);
4243
int F_NO_TRIM_REF_TEXT_START = BitFieldSet.intMask(Flags.NO_TRIM_REF_TEXT_START);
4344
int F_NO_TRIM_REF_TEXT_END = BitFieldSet.intMask(Flags.NO_TRIM_REF_TEXT_END);
45+
int F_ADD_SPACES_BETWEEN_NODES = BitFieldSet.intMask(Flags.ADD_SPACES_BETWEEN_NODES);
4446

4547
/**
4648
* Append node's text

‎flexmark-util-format/src/main/java/com/vladsch/flexmark/util/format/MarkdownParagraph.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -371,7 +371,7 @@ BasedSequence resolveTrackedOffsetsEdit(BasedSequence baseSpliced, BasedSequence
371371

372372
if (addSpacesBefore + addSpacesAfter > 0) {
373373
int lastNonBlank = wrapped.lastIndexOfAnyNot(WHITESPACE_NBSP);
374-
if (wrappedIndex < lastNonBlank) {
374+
if (wrappedIndex <= lastNonBlank) {
375375
// insert in middle
376376
wrapped = wrapped.insert(wrappedIndex, RepeatedSequence.ofSpaces(addSpacesBefore + addSpacesAfter));
377377

‎flexmark-util-sequence/src/main/java/com/vladsch/flexmark/util/sequence/builder/SequenceBuilder.java‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -337,14 +337,6 @@ public String toString() {
337337
BasedSequence s = baseSeq.subSequence(((Range) part).getStart(), ((Range) part).getEnd());
338338

339339
if (s.isNotEmpty()) {
340-
if (last != null && last.isNotEmpty() && last.getEndOffset() < s.getStartOffset()
341-
&& (BasedSequence.WHITESPACE.indexOf(last.charAt(last.length() - 1)) == -1)
342-
&& BasedSequence.WHITESPACE.indexOf(s.charAt(0)) == -1
343-
&& s.baseSubSequence(last.getEndOffset(), s.getStartOffset()).endsWith(" ")
344-
) {
345-
sb.append(' ');
346-
}
347-
348340
s.appendTo(sb);
349341
}
350342

‎flexmark-util-sequence/src/main/java/com/vladsch/flexmark/util/sequence/builder/tree/SegmentTree.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,9 @@ public void addSegments(@NotNull IBasedSegmentBuilder<?> builder, int startIndex
292292
builder.appendAnchor(startOffset);
293293
}
294294

295+
int currentEnd = startOffset;
295296
BasedSequence baseSequence = builder.getBaseSequence();
297+
296298
for (int i = startPos; i < endPos; i++) {
297299
Segment segment = getSegment(i, baseSequence);
298300

@@ -318,12 +320,14 @@ public void addSegments(@NotNull IBasedSegmentBuilder<?> builder, int startIndex
318320
}
319321
} else {
320322
assert charSequence instanceof BasedSequence;
321-
builder.append(((BasedSequence) charSequence).getStartOffset(), ((BasedSequence) charSequence).getEndOffset());
323+
BasedSequence basedSequence = (BasedSequence) charSequence;
324+
currentEnd = Math.max(currentEnd, basedSequence.getEndOffset());
325+
builder.append(basedSequence.getStartOffset(), basedSequence.getEndOffset());
322326
}
323327
}
324328

325329
if (endOffset != -1) {
326-
builder.appendAnchor(endOffset);
330+
builder.appendAnchor(Math.max(currentEnd, endOffset));
327331
}
328332
}
329333

@@ -335,7 +339,6 @@ public void addSegments(@NotNull IBasedSegmentBuilder<?> builder, int startIndex
335339
* @param endIndex end index of sub-sequence of segment tree
336340
* @param startPos start pos of sub-sequence segments in tree
337341
* @param endPos end pos of sub-sequence segments in tree
338-
*
339342
* @return subsequence of segment corresponding to part of it which is in the sub-sequence of the tree
340343
*/
341344
@NotNull
@@ -461,7 +464,7 @@ public static SegmentTreePos findSegmentPos(int index, int[] treeData, int start
461464
}
462465

463466
assert lastStart != startPos || lastEnd != endPos : "Range and position did not change after iteration: pos=" + pos + ", startPos=" + startPos + ", endPos=" + endPos
464-
+ "\n" + Arrays.toString(treeData)
467+
+ "\n" + Arrays.toString(treeData)
465468
;
466469
}
467470
return null;
@@ -531,7 +534,6 @@ public static SegmentTree build(@NotNull BasedSegmentBuilder builder) {
531534
* @param segments segments of the tree
532535
* @param allText all out of base text
533536
* @param buildIndexData true to build index search data, false to build base offset tree data
534-
*
535537
* @return segment tree instance with the data
536538
*/
537539
@NotNull
@@ -619,7 +621,6 @@ public static SegmentTreeData buildTreeData(@NotNull Iterable<Seg> segments, @No
619621
* Efficiently reuses segmentBytes and only computes offset treeData for BASE and ANCHOR segments
620622
*
621623
* @param baseSeq base sequence for the sequence for this segment tree
622-
*
623624
* @return SegmentOffsetTree for this segment tree
624625
*/
625626
@NotNull

0 commit comments

Comments
 (0)