# HG changeset patch
# User Neil <nyamatongwe@gmail.com>
# Date 1522203406 -39600
#      Wed Mar 28 13:16:46 2018 +1100
# Node ID 63808cf2f9853e464f255915b9f75943ea57d0f8
# Parent  d983957decbeb8ca7ed9275b2b79f4a2093e2230
Implement line indexing by UTF-32 code points and UTF-16 code units.

diff -r d983957decbe -r 63808cf2f985 doc/ScintillaDoc.html
--- a/doc/ScintillaDoc.html	Wed Mar 28 12:49:57 2018 +1100
+++ b/doc/ScintillaDoc.html	Wed Mar 28 13:16:46 2018 +1100
@@ -269,136 +269,140 @@
         <tr>
           <td>&cir; <a class="toc" href="#SelectionAndInformation">Selection and information</a></td>
 
+          <td>&cir; <a class="toc" href="#ByCharacterOrCodeUnit">By character or UTF-16 code unit</a></td>
+
           <td>&cir; <a class="toc" href="#MultipleSelectionAndVirtualSpace">Multiple Selection and Virtual Space</a></td>
-
-          <td>&cir; <a class="toc" href="#ScrollingAndAutomaticScrolling">Scrolling and automatic
           scrolling</a></td>
 
         </tr>
 
         <tr>
+          <td>&cir; <a class="toc" href="#ScrollingAndAutomaticScrolling">Scrolling and automatic
+
           <td>&cir; <a class="toc" href="#WhiteSpace">White space</a></td>
 
           <td>&cir; <a class="toc" href="#Cursor">Cursor</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#MouseCapture">Mouse capture</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#LineEndings">Line endings</a></td>
 
           <td>&cir; <a class="toc" href="#Words">Words</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#Styling">Styling</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#StyleDefinition">Style definition</a></td>
 
           <td>&cir; <a class="toc" href="#CaretAndSelectionStyles">Caret, selection, and hotspot styles</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#CharacterRepresentations">Character representations</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#Margins">Margins</a></td>
 
           <td>&cir; <a class="toc" href="#Annotations">Annotations</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#OtherSettings">Other settings</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#BraceHighlighting">Brace highlighting</a></td>
 
           <td>&cir; <a class="toc" href="#TabsAndIndentationGuides">Tabs and Indentation
           Guides</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#Markers">Markers</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#Indicators">Indicators</a></td>
 
           <td>&cir; <a class="toc" href="#Autocompletion">Autocompletion</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#UserLists">User lists</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#CallTips">Call tips</a></td>
 
           <td>&cir; <a class="toc" href="#KeyboardCommands">Keyboard commands</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#KeyBindings">Key bindings</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#PopupEditMenu">Popup edit menu</a></td>
 
           <td>&cir; <a class="toc" href="#MacroRecording">Macro recording</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#Printing">Printing</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#DirectAccess">Direct access</a></td>
 
           <td>&cir; <a class="toc" href="#MultipleViews">Multiple views</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#BackgroundLoadSave">Background loading and saving</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#Folding">Folding</a></td>
 
           <td>&cir; <a class="toc" href="#LineWrapping">Line wrapping</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#Zooming">Zooming</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#LongLines">Long lines</a></td>
 
           <td>&cir; <a class="toc" href="#Accessibility">Accessibility</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#Lexer">Lexer</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#LexerObjects">Lexer objects</a></td>
 
           <td>&cir; <a class="toc" href="#Notifications">Notifications</a></td>
 
+        </tr>
+
+        <tr>
           <td>&cir; <a class="toc" href="#Images">Images</a></td>
 
-        </tr>
-
-        <tr>
           <td>&cir; <a class="toc" href="#GTK">GTK+</a></td>
 
           <td>&cir; <a class="toc" href="#ProvisionalMessages"><span class="provisional">Provisional messages</span></a></td>
 
+        </tr>
+        <tr>
           <td>&cir; <a class="toc" href="#DeprecatedMessages">Deprecated messages</a></td>
 
-        </tr>
-        <tr>
           <td>&cir; <a class="toc" href="#EditMessagesNeverSupportedByScintilla">Edit messages never
           supported by Scintilla</a></td>
 
           <td>&cir; <a class="toc" href="#RemovedFeatures">Removed features</a></td>
 
+        </tr>
+        <tr>
           <td>&cir; <a class="toc" href="#BuildingScintilla">Building Scintilla</a></td>
 
         </tr>
@@ -1228,8 +1232,6 @@
      <a class="message" href="#SCI_MOVECARETINSIDEVIEW">SCI_MOVECARETINSIDEVIEW</a><br />
      <a class="message" href="#SCI_POSITIONBEFORE">SCI_POSITIONBEFORE(int pos) &rarr; position</a><br />
      <a class="message" href="#SCI_POSITIONAFTER">SCI_POSITIONAFTER(int pos) &rarr; position</a><br />
-     <a class="message" href="#SCI_POSITIONRELATIVE">SCI_POSITIONRELATIVE(int pos, int relative) &rarr; position</a><br />
-     <a class="message" href="#SCI_COUNTCHARACTERS">SCI_COUNTCHARACTERS(int start, int end) &rarr; int</a><br />
      <a class="message" href="#SCI_TEXTWIDTH">SCI_TEXTWIDTH(int style, const char *text) &rarr; int</a><br />
      <a class="message" href="#SCI_TEXTHEIGHT">SCI_TEXTHEIGHT(int line) &rarr; int</a><br />
      <a class="message" href="#SCI_CHOOSECARETX">SCI_CHOOSECARETX</a><br />
@@ -1447,15 +1449,6 @@
      If called with a position within a multi byte character will return the position
      of the start/end of that character.</p>
 
-    <p><b id="SCI_POSITIONRELATIVE">SCI_POSITIONRELATIVE(int pos, int relative) &rarr; position</b><br />
-     Count a number of whole characters before or after the argument position and return that position.
-     The minimum position returned is 0 and the maximum is the last position in the document.
-     If the position goes past the document end then 0 is returned.
-     </p>
-
-    <p><b id="SCI_COUNTCHARACTERS">SCI_COUNTCHARACTERS(int start, int end) &rarr; int</b><br />
-     Returns the number of whole characters between two positions..</p>
-
     <p><b id="SCI_TEXTWIDTH">SCI_TEXTWIDTH(int style, const char *text) &rarr; int</b><br />
      This returns the pixel width of a string drawn in the given <code class="parameter">style</code> which can
     be used, for example, to decide how wide to make the line number margin in order to display a
@@ -1526,6 +1519,61 @@
      When this option is turned off, mouse selections will always stick to the mode the selection was started in. It
      is off by default.</p>
 
+    <h2 id="ByCharacterOrCodeUnit">By character or UTF-16 code unit</h2>
+
+    <p>Most Scintilla APIs use byte positions but some applications want to use positions based on counting
+    (UTF-32) characters or (UTF-16) code units
+    or need to communicate with other code written in terms of characters or code units.
+    With only byte positions, this may require examining many bytes to count characters or code units in the document
+    but this may be sped up in some cases by indexing the line starts by character or code unit.</p>
+
+    <code>
+     <a class="message" href="#SCI_POSITIONRELATIVE">SCI_POSITIONRELATIVE(int pos, int relative) &rarr; position</a><br />
+     <a class="message" href="#SCI_POSITIONRELATIVECODEUNITS">SCI_POSITIONRELATIVECODEUNITS(int pos, int relative) &rarr; position</a><br />
+     <a class="message" href="#SCI_COUNTCHARACTERS">SCI_COUNTCHARACTERS(int start, int end) &rarr; int</a><br />
+     <a class="message" href="#SCI_COUNTCODEUNITS">SCI_COUNTCODEUNITS(int start, int end) &rarr; int</a><br />
+     <a class="message" href="#SCI_GETLINECHARACTERINDEX">SCI_GETLINECHARACTERINDEX &rarr; int</a><br />
+     <a class="message" href="#SCI_ALLOCATELINECHARACTERINDEX">SCI_ALLOCATELINECHARACTERINDEX(int lineCharacterIndex)</a><br />
+     <a class="message" href="#SCI_RELEASELINECHARACTERINDEX">SCI_RELEASELINECHARACTERINDEX(int lineCharacterIndex)</a><br />
+     <a class="message" href="#SCI_LINEFROMINDEXPOSITION">SCI_LINEFROMINDEXPOSITION(int pos, int lineCharacterIndex) &rarr; int</a><br />
+     <a class="message" href="#SCI_INDEXPOSITIONFROMLINE">SCI_INDEXPOSITIONFROMLINE(int line, int lineCharacterIndex) &rarr; int</a><br />
+    </code>
+
+    <p><b id="SCI_POSITIONRELATIVE">SCI_POSITIONRELATIVE(int pos, int relative) &rarr; position</b><br />
+     Count a number of whole characters before or after the argument position and return that position.
+     The minimum position returned is 0 and the maximum is the last position in the document.
+     If the position goes past the document end then 0 is returned.
+     </p>
+
+    <p><b id="SCI_COUNTCHARACTERS">SCI_COUNTCHARACTERS(int start, int end) &rarr; int</b><br />
+     Returns the number of whole characters between two positions.</p>
+
+    <p><b id="SCI_POSITIONRELATIVECODEUNITS">SCI_POSITIONRELATIVECODEUNITS(int pos, int relative) &rarr; int</b><br />
+    <b id="SCI_COUNTCODEUNITS">SCI_COUNTCODEUNITS(int start, int end) &rarr; int</b><br />
+     These are the UTF-16 versions of <code>SCI_POSITIONRELATIVE</code> and <code>SCI_COUNTCHARACTERS</code>
+     working in terms of UTF-16 code units.</p>
+
+    <p><b id="SCI_GETLINECHARACTERINDEX">SCI_GETLINECHARACTERINDEX &rarr; int</b><br />
+     Returns which if any indexes are active. It may be <code>SC_LINECHARACTERINDEX_NONE(0)</code> or one or more
+     of <code>SC_LINECHARACTERINDEX_UTF32(1)</code> if whole characters are indexed or
+     <code>SC_LINECHARACTERINDEX_UTF16(2)</code> if UTF-16 code units are indexed.
+     Character indexes are currently only supported for UTF-8 documents.</p>
+
+    <p><b id="SCI_ALLOCATELINECHARACTERINDEX">SCI_ALLOCATELINECHARACTERINDEX(int lineCharacterIndex)</b><br />
+    <b id="SCI_RELEASELINECHARACTERINDEX">SCI_RELEASELINECHARACTERINDEX(int lineCharacterIndex)</b><br />
+     Allocate or release one or more indexes using same enumeration as <code>SCI_GETLINECHARACTERINDEX</code>.
+     Different aspects of an application may need indexes for different periods and should allocate for those periods.
+     Indexes use additional memory so releasing them can help minimize memory but they also take time to recalculate.
+     Scintilla may also allocate indexes to support features like accessibility or input method editors.
+     Only one index of each type is created for a document at a time.</p>
+
+    <p><b id="SCI_LINEFROMINDEXPOSITION">SCI_LINEFROMINDEXPOSITION(int pos, int lineCharacterIndex) &rarr; int</b><br />
+    <b id="SCI_INDEXPOSITIONFROMLINE">SCI_INDEXPOSITIONFROMLINE(int line, int lineCharacterIndex) &rarr; int</b><br />
+     The document line of a particular character or code unit may be found by calling <code>SCI_LINEFROMINDEXPOSITION</code> with one of
+     <code>SC_LINECHARACTERINDEX_UTF32(1)</code> or <code>SC_LINECHARACTERINDEX_UTF16(2)</code>.
+     The inverse action, finds the starting position of a document line either in characters or code units from the document start by calling
+     <code>SCI_INDEXPOSITIONFROMLINE</code> with the same <code class="parameter">lineCharacterIndex</code> argument.</p>
+
     <h2 id="MultipleSelectionAndVirtualSpace">Multiple Selection and Virtual Space</h2>
 
     <code>
diff -r d983957decbe -r 63808cf2f985 include/Scintilla.h
--- a/include/Scintilla.h	Wed Mar 28 12:49:57 2018 +1100
+++ b/include/Scintilla.h	Wed Mar 28 13:16:46 2018 +1100
@@ -368,6 +368,7 @@
 #define SCI_GETLINEINDENTPOSITION 2128
 #define SCI_GETCOLUMN 2129
 #define SCI_COUNTCHARACTERS 2633
+#define SCI_COUNTCODEUNITS 2715
 #define SCI_SETHSCROLLBAR 2130
 #define SCI_GETHSCROLLBAR 2131
 #define SC_IV_NONE 0
@@ -753,6 +754,7 @@
 #define SCI_POSITIONBEFORE 2417
 #define SCI_POSITIONAFTER 2418
 #define SCI_POSITIONRELATIVE 2670
+#define SCI_POSITIONRELATIVECODEUNITS 2716
 #define SCI_COPYRANGE 2419
 #define SCI_COPYTEXT 2420
 #define SC_SEL_STREAM 0
@@ -1112,6 +1114,14 @@
 #define SC_BIDIRECTIONAL_R2L 2
 #define SCI_GETBIDIRECTIONAL 2708
 #define SCI_SETBIDIRECTIONAL 2709
+#define SC_LINECHARACTERINDEX_NONE 0
+#define SC_LINECHARACTERINDEX_UTF32 1
+#define SC_LINECHARACTERINDEX_UTF16 2
+#define SCI_GETLINECHARACTERINDEX 2710
+#define SCI_ALLOCATELINECHARACTERINDEX 2711
+#define SCI_RELEASELINECHARACTERINDEX 2712
+#define SCI_LINEFROMINDEXPOSITION 2713
+#define SCI_INDEXPOSITIONFROMLINE 2714
 #endif
 /* --Autogenerated -- end of section automatically generated from Scintilla.iface */
 
diff -r d983957decbe -r 63808cf2f985 include/Scintilla.iface
--- a/include/Scintilla.iface	Wed Mar 28 12:49:57 2018 +1100
+++ b/include/Scintilla.iface	Wed Mar 28 13:16:46 2018 +1100
@@ -860,6 +860,9 @@
 # Count characters between two positions.
 fun int CountCharacters=2633(position start, position end)
 
+# Count code units between two positions.
+fun int CountCodeUnits=2715(position start, position end)
+
 # Show or hide the horizontal scroll bar.
 set void SetHScrollBar=2130(bool visible,)
 # Is the horizontal scroll bar visible?
@@ -1950,6 +1953,11 @@
 # of characters. Returned value is always between 0 and last position in document.
 fun position PositionRelative=2670(position pos, int relative)
 
+# Given a valid document position, return a position that differs in a number
+# of UTF-16 code units. Returned value is always between 0 and last position in document.
+# The result may point half way (2 bytes) inside a non-BMP character.
+fun position PositionRelativeCodeUnits=2716(position pos, int relative)
+
 # Copy a range of text to the clipboard. Positions are clipped into the document.
 fun void CopyRange=2419(position start, position end)
 
@@ -4890,6 +4898,26 @@
 # Set bidirectional text display state.
 set void SetBidirectional=2709(int bidirectional,)
 
+enu LineCharacterIndex=SC_LINECHARACTERINDEX_
+val SC_LINECHARACTERINDEX_NONE=0
+val SC_LINECHARACTERINDEX_UTF32=1
+val SC_LINECHARACTERINDEX_UTF16=2
+
+# Retrieve line character index state.
+get int GetLineCharacterIndex=2710(,)
+
+# Request line character index be created or its use count increased.
+fun void AllocateLineCharacterIndex=2711(int lineCharacterIndex,)
+
+# Decrease use count of line character index and remove if 0.
+fun void ReleaseLineCharacterIndex=2712(int lineCharacterIndex,)
+
+# Retrieve the document line containing a position measured in index units.
+fun int LineFromIndexPosition=2713(position posUTF32, int lineCharacterIndex)
+
+# Retrieve the position measured in index units at the start of a document line.
+fun position IndexPositionFromLine=2714(int line, int lineCharacterIndex)
+
 cat Deprecated
 
 # Divide each styling byte into lexical class bits (default: 5) and indicator
diff -r d983957decbe -r 63808cf2f985 src/CellBuffer.cxx
--- a/src/CellBuffer.cxx	Wed Mar 28 12:49:57 2018 +1100
+++ b/src/CellBuffer.cxx	Wed Mar 28 13:16:46 2018 +1100
@@ -7,6 +7,7 @@
 
 #include <cstddef>
 #include <cstdlib>
+#include <cassert>
 #include <cstring>
 #include <cstdio>
 #include <cstdarg>
@@ -40,6 +41,13 @@
 	virtual Sci::Line Lines() const = 0;
 	virtual Sci::Line LineFromPosition(Sci::Position pos) const = 0;
 	virtual Sci::Position LineStart(Sci::Line line) const = 0;
+	virtual void InsertCharacters(Sci::Line line, CountWidths delta) = 0;
+	virtual void SetLineCharactersWidth(Sci::Line line, CountWidths width) = 0;
+	virtual int LineCharacterIndex() const = 0;
+	virtual bool AllocateLineCharacterIndex(int lineCharacterIndex, Sci::Line lines) = 0;
+	virtual bool ReleaseLineCharacterIndex(int lineCharacterIndex) = 0;
+	virtual Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const = 0;
+	virtual Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const = 0;
 	virtual ~ILineVector() {}
 };
 
@@ -47,10 +55,54 @@
 
 using namespace Scintilla;
 
+LineStartIndex::LineStartIndex() : refCount(0), starts(4) {
+	// Minimal initial allocation
+}
+
+LineStartIndex::~LineStartIndex() {
+	starts.DeleteAll();
+}
+
+bool LineStartIndex::Allocate(Sci::Line lines) {
+	refCount++;
+	Sci::Position length = starts.PositionFromPartition(starts.Partitions());
+	for (Sci::Line line = starts.Partitions(); line < lines; line++) {
+		// Produce an ascending sequence that will be filled in with correct widths later
+		length++;
+		starts.InsertPartition(line, length);
+	}
+	return refCount == 1;
+}
+
+bool LineStartIndex::Release() {
+	if (refCount == 1) {
+		starts.DeleteAll();
+	}
+	refCount--;
+	return refCount == 0;
+}
+
+bool LineStartIndex::Active() const {
+	return refCount > 0;
+}
+
+Sci::Position LineStartIndex::LineWidth(Sci::Line line) const {
+	return starts.PositionFromPartition(line + 1) - starts.PositionFromPartition(line);
+}
+
+void LineStartIndex::SetLineWidth(Sci::Line line, Sci::Position width) {
+	const Sci::Position widthCurrent = LineWidth(line);
+	starts.InsertText(line, width - widthCurrent);
+}
+
 template <typename POS>
 class LineVector : public ILineVector {
 	Partitioning<POS> starts;
 	PerLine *perLine;
+
+	LineStartIndex startsUTF16;
+	LineStartIndex startsUTF32;
+
 public:
 	LineVector() : starts(256), perLine(0) {
 		Init();
@@ -72,6 +124,12 @@
 	}
 	void InsertLine(Sci::Line line, Sci::Position position, bool lineStart) override {
 		starts.InsertPartition(static_cast<POS>(line), static_cast<POS>(position));
+		if (startsUTF32.Active()) {
+			startsUTF32.starts.InsertPartition(line, startsUTF32.starts.PositionFromPartition(line - 1) + 1);
+		}
+		if (startsUTF16.Active()) {
+			startsUTF16.starts.InsertPartition(line, startsUTF16.starts.PositionFromPartition(line - 1) + 1);
+		}
 		if (perLine) {
 			if ((line > 0) && lineStart)
 				line--;
@@ -83,6 +141,12 @@
 	}
 	void RemoveLine(Sci::Line line) override {
 		starts.RemovePartition(static_cast<POS>(line));
+		if (startsUTF32.Active()) {
+			startsUTF32.starts.RemovePartition(line);
+		}
+		if (startsUTF16.Active()) {
+			startsUTF16.starts.RemovePartition(line);
+		}
 		if (perLine) {
 			perLine->RemoveLine(line);
 		}
@@ -96,6 +160,70 @@
 	Sci::Position LineStart(Sci::Line line) const override {
 		return starts.PositionFromPartition(static_cast<POS>(line));
 	}
+	void InsertCharacters(Sci::Line line, CountWidths delta) override {
+		if (startsUTF32.Active()) {
+			startsUTF32.starts.InsertText(line, delta.WidthUTF32());
+		}
+		if (startsUTF16.Active()) {
+			startsUTF16.starts.InsertText(line, delta.WidthUTF16());
+		}
+	}
+	void SetLineCharactersWidth(Sci::Line line, CountWidths width) override {
+		if (startsUTF32.Active()) {
+			assert(startsUTF32.starts.Partitions() == starts.Partitions());
+			startsUTF32.SetLineWidth(line, width.WidthUTF32());
+		}
+		if (startsUTF16.Active()) {
+			assert(startsUTF16.starts.Partitions() == starts.Partitions());
+			startsUTF16.SetLineWidth(line, width.WidthUTF16());
+		}
+	}
+	int LineCharacterIndex() const override {
+		int retVal = 0;
+		if (startsUTF32.Active()) {
+			retVal |= SC_LINECHARACTERINDEX_UTF32;
+		}
+		if (startsUTF16.Active()) {
+			retVal |= SC_LINECHARACTERINDEX_UTF16;
+		}
+		return retVal;
+	}
+	bool AllocateLineCharacterIndex(int lineCharacterIndex, Sci::Line lines) override {
+		bool changed = false;
+		if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF32) != 0) {
+			changed = startsUTF32.Allocate(lines) || changed;
+			assert(startsUTF32.starts.Partitions() == starts.Partitions());
+		}
+		if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF16) != 0) {
+			changed = startsUTF16.Allocate(lines) || changed;
+			assert(startsUTF16.starts.Partitions() == starts.Partitions());
+		}
+		return changed;
+	}
+	bool ReleaseLineCharacterIndex(int lineCharacterIndex) override {
+		bool changed = false;
+		if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF32) != 0) {
+			changed = startsUTF32.Release() || changed;
+		}
+		if ((lineCharacterIndex & SC_LINECHARACTERINDEX_UTF16) != 0) {
+			changed = startsUTF16.Release() || changed;
+		}
+		return changed;
+	}
+	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const override {
+		if (lineCharacterIndex == SC_LINECHARACTERINDEX_UTF32) {
+			return startsUTF32.starts.PositionFromPartition(line);
+		} else {
+			return startsUTF16.starts.PositionFromPartition(line);
+		}
+	}
+	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const override {
+		if (lineCharacterIndex == SC_LINECHARACTERINDEX_UTF32) {
+			return startsUTF32.starts.PartitionFromPosition(pos);
+		} else {
+			return startsUTF16.starts.PartitionFromPosition(pos);
+		}
+	}
 };
 
 Action::Action() {
@@ -173,6 +301,8 @@
 		// Run out of undo nodes so extend the array
 		actions.resize(actions.size() * 2);
 	}
+	startsUTF32.starts.DeleteAll();
+	startsUTF16.starts.DeleteAll();
 }
 
 const char *UndoHistory::AppendAction(actionType at, Sci::Position position, const char *data, Sci::Position lengthData,
@@ -369,6 +499,7 @@
 CellBuffer::CellBuffer(bool hasStyles_, bool largeDocument_) :
 	hasStyles(hasStyles_), largeDocument(largeDocument_) {
 	readOnly = false;
+	utf8Substance = false;
 	utf8LineEnds = 0;
 	collectingUndo = true;
 	if (largeDocument)
@@ -506,10 +637,19 @@
 	}
 }
 
+void CellBuffer::SetUTF8Substance(bool utf8Substance_) {
+	if (utf8Substance != utf8Substance_) {
+		utf8Substance = utf8Substance_;
+		ResetLineEnds();
+	}
+}
+
 void CellBuffer::SetLineEndTypes(int utf8LineEnds_) {
 	if (utf8LineEnds != utf8LineEnds_) {
+		const int indexes = plv->LineCharacterIndex();
 		utf8LineEnds = utf8LineEnds_;
 		ResetLineEnds();
+		AllocateLineCharacterIndex(indexes);
 	}
 }
 
@@ -536,6 +676,23 @@
 	plv->SetPerLine(pl);
 }
 
+int CellBuffer::LineCharacterIndex() const {
+	return plv->LineCharacterIndex();
+}
+
+void CellBuffer::AllocateLineCharacterIndex(int lineCharacterIndex) {
+	if (utf8Substance) {
+		if (plv->AllocateLineCharacterIndex(lineCharacterIndex, Lines())) {
+			// Changed so recalculate whole file
+			RecalculateIndexLineStarts(0, Lines() - 1);
+		}
+	}
+}
+
+void CellBuffer::ReleaseLineCharacterIndex(int lineCharacterIndex) {
+	plv->ReleaseLineCharacterIndex(lineCharacterIndex);
+}
+
 Sci::Line CellBuffer::Lines() const {
 	return plv->Lines();
 }
@@ -553,6 +710,14 @@
 	return plv->LineFromPosition(pos);
 }
 
+Sci::Position CellBuffer::IndexLineStart(Sci::Line line, int lineCharacterIndex) const {
+	return plv->IndexLineStart(line, lineCharacterIndex);
+}
+
+Sci::Line CellBuffer::LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const {
+	return plv->LineFromPositionIndex(pos, lineCharacterIndex);
+}
+
 bool CellBuffer::IsReadOnly() const {
 	return readOnly;
 }
@@ -609,6 +774,37 @@
 	return UTF8IsSeparator(bytes) || UTF8IsSeparator(bytes+1) || UTF8IsNEL(bytes+1);
 }
 
+bool CellBuffer::UTF8IsCharacterBoundary(Sci::Position position) const {
+	assert(position >= 0 && position <= Length());
+	if (position > 0) {
+		std::string back;
+		for (int i = 0; i < UTF8MaxBytes; i++) {
+			const Sci::Position posBack = position - i;
+			if (posBack < 0) {
+				return false;
+			}
+			back.insert(0, 1, substance.ValueAt(posBack));
+			if (!UTF8IsTrailByte(back.front())) {
+				if (i > 0) {
+					// Have reached a non-trail
+					const int cla = UTF8Classify(reinterpret_cast<const unsigned char *>(back.c_str()), i);
+					if ((cla & UTF8MaskInvalid) || (cla != i)) {
+						return false;
+					}
+				}
+				break;
+			}
+		}
+	}
+	if (position < Length()) {
+		const unsigned char fore = static_cast<unsigned char>(substance.ValueAt(position));
+		if (UTF8IsTrailByte(fore)) {
+			return false;
+		}
+	}
+	return true;
+}
+
 void CellBuffer::ResetLineEnds() {
 	// Reinitialize line data -- too much work to preserve
 	plv->Init();
@@ -645,6 +841,39 @@
 	}
 }
 
+namespace {
+
+CountWidths CountCharacterWidthsUTF8(const char *s, Sci::Position len) {
+	const unsigned char *us = reinterpret_cast<const unsigned char *>(s);
+	CountWidths cw;
+	size_t remaining = len;
+	while (remaining > 0) {
+		const int utf8Status = UTF8Classify(us, static_cast<int>(remaining));
+		const int lenChar = utf8Status & UTF8MaskWidth;
+		cw.CountChar(lenChar);
+		us += lenChar;
+		remaining -= lenChar;
+	}
+	return cw;
+}
+
+}
+
+void CellBuffer::RecalculateIndexLineStarts(Sci::Line lineFirst, Sci::Line lineLast) {
+	std::vector<char> text;
+	Sci::Position posLineEnd = LineStart(lineFirst);
+	for (Sci::Line line = lineFirst; line <= lineLast; line++) {
+		// Find line start and end, retrieve text of line, count characters and update line width
+		const Sci::Position posLineStart = posLineEnd;
+		posLineEnd = LineStart(line+1);
+		const Sci::Position width = posLineEnd - posLineStart;
+		text.resize(width);
+		GetCharRange(text.data(), posLineStart, width);
+		const CountWidths cw = CountCharacterWidthsUTF8(text.data(), width);
+		plv->SetLineCharactersWidth(line, cw);
+	}
+}
+
 void CellBuffer::BasicInsertString(Sci::Position position, const char *s, Sci::Position insertLength) {
 	if (insertLength == 0)
 		return;
@@ -656,12 +885,25 @@
 		breakingUTF8LineEnd = UTF8LineEndOverlaps(position);
 	}
 
+	const Sci::Line linePosition = plv->LineFromPosition(position);
+	Sci::Line lineInsert = linePosition + 1;
+
+	// A simple insertion is one that inserts valid text on a single line at a character boundary
+	bool simpleInsertion = false;
+
+	// Check for breaking apart a UTF-8 sequence and inserting invalid UTF-8
+	if (utf8Substance && (plv->LineCharacterIndex() != SC_LINECHARACTERINDEX_NONE)) {
+		// Actually, don't need to check that whole insertion is valid just that there
+		// are no potential fragments at ends.
+		simpleInsertion = UTF8IsCharacterBoundary(position) &&
+			UTF8IsValid(std::string_view(s, insertLength));
+	}
+
 	substance.InsertFromArray(position, s, 0, insertLength);
 	if (hasStyles) {
 		style.InsertValue(position, insertLength, 0);
 	}
 
-	Sci::Line lineInsert = plv->LineFromPosition(position) + 1;
 	const bool atLineStart = plv->LineStart(lineInsert-1) == position;
 	// Point all the lines after the insertion point further along in the buffer
 	plv->InsertText(lineInsert-1, insertLength);
@@ -681,6 +923,7 @@
 		if (ch == '\r') {
 			InsertLine(lineInsert, (position + i) + 1, atLineStart);
 			lineInsert++;
+			simpleInsertion = false;
 		} else if (ch == '\n') {
 			if (chPrev == '\r') {
 				// Patch up what was end of line
@@ -689,11 +932,13 @@
 				InsertLine(lineInsert, (position + i) + 1, atLineStart);
 				lineInsert++;
 			}
+			simpleInsertion = false;
 		} else if (utf8LineEnds) {
 			const unsigned char back3[3] = {chBeforePrev, chPrev, ch};
 			if (UTF8IsSeparator(back3) || UTF8IsNEL(back3+1)) {
 				InsertLine(lineInsert, (position + i) + 1, atLineStart);
 				lineInsert++;
+				simpleInsertion = false;
 			}
 		}
 		chBeforePrev = chPrev;
@@ -704,6 +949,7 @@
 		if (ch == '\r') {
 			// End of line already in buffer so drop the newly created one
 			RemoveLine(lineInsert - 1);
+			simpleInsertion = false;
 		}
 	} else if (utf8LineEnds && !UTF8IsAscii(chAfter)) {
 		// May have end of UTF-8 line end in buffer and start in insertion
@@ -713,21 +959,31 @@
 			if (UTF8IsSeparator(back3)) {
 				InsertLine(lineInsert, (position + insertLength + j) + 1, atLineStart);
 				lineInsert++;
+				simpleInsertion = false;
 			}
 			if ((j == 0) && UTF8IsNEL(back3+1)) {
 				InsertLine(lineInsert, (position + insertLength + j) + 1, atLineStart);
 				lineInsert++;
+				simpleInsertion = false;
 			}
 			chBeforePrev = chPrev;
 			chPrev = chAt;
 		}
 	}
+	if (simpleInsertion) {
+		const CountWidths cw = CountCharacterWidthsUTF8(s, insertLength);
+		plv->InsertCharacters(linePosition, cw);
+	} else {
+		RecalculateIndexLineStarts(linePosition, lineInsert - 1);
+	}
 }
 
 void CellBuffer::BasicDeleteChars(Sci::Position position, Sci::Position deleteLength) {
 	if (deleteLength == 0)
 		return;
 
+	Sci::Line lineRecalculateStart = INVALID_POSITION;
+
 	if ((position == 0) && (deleteLength == substance.Length())) {
 		// If whole buffer is being deleted, faster to reinitialise lines data
 		// than to delete each line.
@@ -736,11 +992,36 @@
 		// Have to fix up line positions before doing deletion as looking at text in buffer
 		// to work out which lines have been removed
 
-		Sci::Line lineRemove = plv->LineFromPosition(position) + 1;
+		const Sci::Line linePosition = plv->LineFromPosition(position);
+		Sci::Line lineRemove = linePosition + 1;
 		plv->InsertText(lineRemove-1, - (deleteLength));
 		const unsigned char chPrev = substance.ValueAt(position - 1);
 		const unsigned char chBefore = chPrev;
 		unsigned char chNext = substance.ValueAt(position);
+
+		// Check for breaking apart a UTF-8 sequence
+		// Needs further checks that text is UTF-8 or that some other break apart is occurring
+		if (utf8Substance && (plv->LineCharacterIndex() != SC_LINECHARACTERINDEX_NONE)) {
+			const Sci::Position posEnd = position + deleteLength;
+			const Sci::Line lineEndRemove = plv->LineFromPosition(posEnd);
+			const bool simpleDeletion =
+				(linePosition == lineEndRemove) &&
+				UTF8IsCharacterBoundary(position) && UTF8IsCharacterBoundary(posEnd);
+			if (simpleDeletion) {
+				std::string text(deleteLength, '\0');
+				GetCharRange(text.data(), position, deleteLength);
+				if (UTF8IsValid(text)) {
+					// Everything is good
+					const CountWidths cw = CountCharacterWidthsUTF8(text.data(), deleteLength);
+					plv->InsertCharacters(linePosition, -cw);
+				} else {
+					lineRecalculateStart = linePosition;
+				}
+			} else {
+				lineRecalculateStart = linePosition;
+			}
+		}
+
 		bool ignoreNL = false;
 		if (chPrev == '\r' && chNext == '\n') {
 			// Move back one
@@ -789,6 +1070,9 @@
 		}
 	}
 	substance.DeleteRange(position, deleteLength);
+	if (lineRecalculateStart >= 0) {
+		RecalculateIndexLineStarts(lineRecalculateStart, lineRecalculateStart);
+	}
 	if (hasStyles) {
 		style.DeleteRange(position, deleteLength);
 	}
diff -r d983957decbe -r 63808cf2f985 src/CellBuffer.h
--- a/src/CellBuffer.h	Wed Mar 28 12:49:57 2018 +1100
+++ b/src/CellBuffer.h	Wed Mar 28 13:16:46 2018 +1100
@@ -19,6 +19,58 @@
 	virtual void RemoveLine(Sci::Line line)=0;
 };
 
+struct CountWidths {
+	// Measures the number of characters in a string divided into those
+	// from the Base Multilingual Plane and those from other planes.
+	Sci::Position countBasePlane;
+	Sci::Position countOtherPlanes;
+	CountWidths(Sci::Position countBasePlane_=0, Sci::Position countOtherPlanes_=0) :
+		countBasePlane(countBasePlane_),
+		countOtherPlanes(countOtherPlanes_) {
+	}
+	CountWidths operator-() const {
+		return CountWidths(-countBasePlane , -countOtherPlanes);
+	}
+	Sci::Position WidthUTF32() const {
+		// All code points take one code unit in UTF-32.
+		return countBasePlane + countOtherPlanes;
+	}
+	Sci::Position WidthUTF16() const {
+		// UTF-16 takes 2 code units for other planes
+		return countBasePlane + 2 * countOtherPlanes;
+	}
+	void CountChar(int lenChar) {
+		if (lenChar == 4) {
+			countOtherPlanes++;
+		} else {
+			countBasePlane++;
+		}
+	}
+
+	void InsertCharacters(Sci::Line line, CountWidths delta);
+	void SetLineCharactersWidth(Sci::Line line, CountWidths width);
+
+	int LineCharacterIndex() const;
+	bool AllocateLineCharacterIndex(int lineCharacterIndex, Sci::Line lines);
+	bool ReleaseLineCharacterIndex(int lineCharacterIndex);
+	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const;
+	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const;
+};
+
+class LineStartIndex {
+public:
+	int refCount;
+	Partitioning<Sci::Position> starts;
+
+	LineStartIndex();
+	virtual ~LineStartIndex();
+	bool Allocate(Sci::Line lines);
+	bool Release();
+	bool Active() const;
+	Sci::Position LineWidth(Sci::Line line) const;
+	void SetLineWidth(Sci::Line line, Sci::Position width);
+};
+
 /**
  * The line vector contains information about each of the lines in a cell buffer.
  */
@@ -63,6 +115,9 @@
 
 	void EnsureUndoRoom();
 
+	LineStartIndex startsUTF16;
+	LineStartIndex startsUTF32;
+
 public:
 	UndoHistory();
 	// Deleted so UndoHistory objects can not be copied.
@@ -112,6 +167,7 @@
 	SplitVector<char> substance;
 	SplitVector<char> style;
 	bool readOnly;
+	bool utf8Substance;
 	int utf8LineEnds;
 
 	bool collectingUndo;
@@ -120,7 +176,9 @@
 	std::unique_ptr<ILineVector> plv;
 
 	bool UTF8LineEndOverlaps(Sci::Position position) const;
+	bool UTF8IsCharacterBoundary(Sci::Position position) const;
 	void ResetLineEnds();
+	void RecalculateIndexLineStarts(Sci::Line lineFirst, Sci::Line lineLast);
 	/// Actions without undo
 	void BasicInsertString(Sci::Position position, const char *s, Sci::Position insertLength);
 	void BasicDeleteChars(Sci::Position position, Sci::Position deleteLength);
@@ -144,13 +202,19 @@
 
 	Sci::Position Length() const;
 	void Allocate(Sci::Position newSize);
+	void SetUTF8Substance(bool utf8Substance_);
 	int GetLineEndTypes() const { return utf8LineEnds; }
 	void SetLineEndTypes(int utf8LineEnds_);
 	bool ContainsLineEnd(const char *s, Sci::Position length) const;
 	void SetPerLine(PerLine *pl);
+	int LineCharacterIndex() const;
+	void AllocateLineCharacterIndex(int lineCharacterIndex);
+	void ReleaseLineCharacterIndex(int lineCharacterIndex);
 	Sci::Line Lines() const;
 	Sci::Position LineStart(Sci::Line line) const;
+	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const;
 	Sci::Line LineFromPosition(Sci::Position pos) const;
+	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const;
 	void InsertLine(Sci::Line line, Sci::Position position, bool lineStart);
 	void RemoveLine(Sci::Line line);
 	const char *InsertString(Sci::Position position, const char *s, Sci::Position insertLength, bool &startSequence);
diff -r d983957decbe -r 63808cf2f985 src/Document.cxx
--- a/src/Document.cxx	Wed Mar 28 12:49:57 2018 +1100
+++ b/src/Document.cxx	Wed Mar 28 13:16:46 2018 +1100
@@ -125,6 +125,7 @@
 	decorations = DecorationListCreate(true);
 
 	cb.SetPerLine(this);
+	cb.SetUTF8Substance(SC_CP_UTF8 == dbcsCodePage);
 }
 
 Document::~Document() {
@@ -172,6 +173,7 @@
 		dbcsCodePage = dbcsCodePage_;
 		SetCaseFolder(nullptr);
 		cb.SetLineEndTypes(lineEndBitSet & LineEndTypesSupported());
+		cb.SetUTF8Substance(SC_CP_UTF8 == dbcsCodePage);
 		return true;
 	} else {
 		return false;
@@ -421,6 +423,14 @@
 		return startText;
 }
 
+Sci::Position Document::IndexLineStart(Sci::Line line, int lineCharacterIndex) const {
+	return cb.IndexLineStart(line, lineCharacterIndex);
+}
+
+Sci::Line Document::LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const {
+	return cb.LineFromPositionIndex(pos, lineCharacterIndex);
+}
+
 int SCI_METHOD Document::SetLevel(Sci_Position line, int level) {
 	const int prev = Levels()->SetLevel(static_cast<Sci::Line>(line), level, LinesTotal());
 	if (prev != level) {
@@ -2014,6 +2024,18 @@
 		return 0;
 }
 
+int Document::LineCharacterIndex() const {
+	return cb.LineCharacterIndex();
+}
+
+void Document::AllocateLineCharacterIndex(int lineCharacterIndex) {
+	return cb.AllocateLineCharacterIndex(lineCharacterIndex);
+}
+
+void Document::ReleaseLineCharacterIndex(int lineCharacterIndex) {
+	return cb.ReleaseLineCharacterIndex(lineCharacterIndex);
+}
+
 Sci::Line Document::LinesTotal() const {
 	return cb.Lines();
 }
diff -r d983957decbe -r 63808cf2f985 src/Document.h
--- a/src/Document.h	Wed Mar 28 12:49:57 2018 +1100
+++ b/src/Document.h	Wed Mar 28 13:16:46 2018 +1100
@@ -381,6 +381,8 @@
 	bool IsLineEndPosition(Sci::Position position) const;
 	bool IsPositionInLineEnd(Sci::Position position) const;
 	Sci::Position VCHomePosition(Sci::Position position) const;
+	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const;
+	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const;
 
 	int SCI_METHOD SetLevel(Sci_Position line, int level);
 	int SCI_METHOD GetLevel(Sci_Position line) const;
@@ -406,6 +408,9 @@
 	void SetCaseFolder(CaseFolder *pcf_);
 	Sci::Position FindText(Sci::Position minPos, Sci::Position maxPos, const char *search, int flags, Sci::Position *length);
 	const char *SubstituteByPosition(const char *text, Sci::Position *length);
+	int LineCharacterIndex() const;
+	void AllocateLineCharacterIndex(int lineCharacterIndex);
+	void ReleaseLineCharacterIndex(int lineCharacterIndex);
 	Sci::Line LinesTotal() const;
 
 	void SetDefaultCharClasses(bool includeWordClass);
diff -r d983957decbe -r 63808cf2f985 src/Editor.cxx
--- a/src/Editor.cxx	Wed Mar 28 12:49:57 2018 +1100
+++ b/src/Editor.cxx	Wed Mar 28 13:16:46 2018 +1100
@@ -6021,8 +6021,12 @@
 		return pdoc->MovePositionOutsideChar(static_cast<int>(wParam) + 1, 1, true);
 
 	case SCI_POSITIONRELATIVE:
-		return std::clamp(static_cast<int>(pdoc->GetRelativePosition(static_cast<int>(wParam), static_cast<int>(lParam))),
-			0, static_cast<int>(pdoc->Length()));
+		return std::clamp(pdoc->GetRelativePosition(static_cast<Sci_Position>(wParam), static_cast<Sci_Position>(lParam)),
+			static_cast<Sci_Position>(0), pdoc->Length());
+
+	case SCI_POSITIONRELATIVECODEUNITS:
+		return std::clamp(pdoc->GetRelativePositionUTF16(static_cast<Sci::Position>(wParam), static_cast<Sci::Position>(lParam)),
+			static_cast<Sci::Position>(0), static_cast<Sci::Position>(pdoc->Length()));
 
 	case SCI_LINESCROLL:
 		ScrollTo(topLine + static_cast<Sci::Line>(lParam));
@@ -6310,6 +6314,7 @@
 
 	case SCI_REDO:
 		Redo();
+pdoc->AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16);
 		break;
 
 	case SCI_SELECTALL:
@@ -6789,6 +6794,23 @@
 	case SCI_GETBIDIRECTIONAL:
 		return static_cast<sptr_t>(bidirectional);
 
+	case SCI_GETLINECHARACTERINDEX:
+		return pdoc->LineCharacterIndex();
+
+	case SCI_ALLOCATELINECHARACTERINDEX:
+		pdoc->AllocateLineCharacterIndex(static_cast<int>(wParam));
+		break;
+
+	case SCI_RELEASELINECHARACTERINDEX:
+		pdoc->ReleaseLineCharacterIndex(static_cast<int>(wParam));
+		break;
+
+	case SCI_LINEFROMINDEXPOSITION:
+		return pdoc->LineFromPositionIndex(static_cast<Sci::Position>(wParam), static_cast<int>(lParam));
+
+	case SCI_INDEXPOSITIONFROMLINE:
+		return pdoc->IndexLineStart(static_cast<Sci::Line>(wParam), static_cast<int>(lParam));
+
 		// Marker definition and setting
 	case SCI_MARKERDEFINE:
 		if (wParam <= MARKER_MAX) {
@@ -8192,7 +8214,10 @@
 		return technology;
 
 	case SCI_COUNTCHARACTERS:
-		return pdoc->CountCharacters(static_cast<int>(wParam), static_cast<int>(lParam));
+		return pdoc->CountCharacters(static_cast<Sci::Position>(wParam), static_cast<Sci::Position>(lParam));
+
+	case SCI_COUNTCODEUNITS:
+		return pdoc->CountUTF16(static_cast<Sci::Position>(wParam), static_cast<Sci::Position>(lParam));
 
 	default:
 		return DefWndProc(iMessage, wParam, lParam);
diff -r d983957decbe -r 63808cf2f985 src/UniConversion.cxx
--- a/src/UniConversion.cxx	Wed Mar 28 12:49:57 2018 +1100
+++ b/src/UniConversion.cxx	Wed Mar 28 13:16:46 2018 +1100
@@ -331,6 +331,22 @@
 	return (utf8StatusNext & UTF8MaskInvalid) ? 1 : (utf8StatusNext & UTF8MaskWidth);
 }
 
+bool UTF8IsValid(std::string_view sv) {
+	const unsigned char *us = reinterpret_cast<const unsigned char *>(sv.data());
+	size_t remaining = sv.length();
+	while (remaining > 0) {
+		const int utf8Status = UTF8Classify(us, static_cast<int>(remaining));
+		if (utf8Status & UTF8MaskInvalid) {
+			return false;
+		} else {
+			const int lenChar = utf8Status & UTF8MaskWidth;
+			us += lenChar;
+			remaining -= lenChar;
+		}
+	}
+	return remaining == 0;
+}
+
 // Replace invalid bytes in UTF-8 with the replacement character
 std::string FixInvalidUTF8(const std::string &text) {
 	std::string result;
diff -r d983957decbe -r 63808cf2f985 src/UniConversion.h
--- a/src/UniConversion.h	Wed Mar 28 12:49:57 2018 +1100
+++ b/src/UniConversion.h	Wed Mar 28 13:16:46 2018 +1100
@@ -21,6 +21,7 @@
 size_t UTF16FromUTF8(const char *s, size_t len, wchar_t *tbuf, size_t tlen);
 size_t UTF32FromUTF8(const char *s, size_t len, unsigned int *tbuf, size_t tlen);
 unsigned int UTF16FromUTF32Character(unsigned int val, wchar_t *tbuf);
+bool UTF8IsValid(std::string_view sv);
 std::string FixInvalidUTF8(const std::string &text);
 
 extern const unsigned char UTF8BytesOfLead[256];
diff -r d983957decbe -r 63808cf2f985 test/unit/testCellBuffer.cxx
--- a/test/unit/testCellBuffer.cxx	Wed Mar 28 12:49:57 2018 +1100
+++ b/test/unit/testCellBuffer.cxx	Wed Mar 28 13:16:46 2018 +1100
@@ -9,6 +9,7 @@
 
 #include "Platform.h"
 
+#include "Scintilla.h"
 #include "Position.h"
 #include "SplitVector.h"
 #include "Partitioning.h"
@@ -144,3 +145,290 @@
 	}
 
 }
+
+TEST_CASE("Character Index") {
+
+	CellBuffer cb(true, false);
+
+	SECTION("Setup") {
+		REQUIRE(cb.LineCharacterIndex() == SC_LINECHARACTERINDEX_NONE);
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 0);
+		cb.SetUTF8Substance(true);
+
+		cb.AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16);
+		REQUIRE(cb.LineCharacterIndex() == SC_LINECHARACTERINDEX_UTF16);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 0);
+
+		cb.ReleaseLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16);
+		REQUIRE(cb.LineCharacterIndex() == SC_LINECHARACTERINDEX_NONE);
+	}
+
+	SECTION("Insertion") {
+		cb.SetUTF8Substance(true);
+
+		cb.AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16 | SC_LINECHARACTERINDEX_UTF32);
+
+		bool startSequence = false;
+		cb.InsertString(0, "a", 1, startSequence);
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 1);
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 1);
+
+		const char *hwair = "\xF0\x90\x8D\x88";
+		cb.InsertString(0, hwair, strlen(hwair), startSequence);
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 3);
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 2);
+	}
+
+	SECTION("Deletion") {
+		cb.SetUTF8Substance(true);
+
+		cb.AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16 | SC_LINECHARACTERINDEX_UTF32);
+
+		bool startSequence = false;
+		const char *hwair = "a\xF0\x90\x8D\x88z";
+		cb.InsertString(0, hwair, strlen(hwair), startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 4);
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 3);
+
+		cb.DeleteChars(5, 1, startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 3);
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 2);
+
+		cb.DeleteChars(1, 4, startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 1);
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 1);
+	}
+
+	SECTION("Insert Complex") {
+		cb.SetUTF8Substance(true);
+		cb.SetLineEndTypes(1);
+		cb.AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16 | SC_LINECHARACTERINDEX_UTF32);
+
+		bool startSequence = false;
+		// 3 lines of text containing 8 bytes
+		const char *data = "a\n\xF0\x90\x8D\x88\nz";
+		cb.InsertString(0, data, strlen(data), startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 5);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF16) == 6);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 4);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF32) == 5);
+
+		// Insert a new line at end -> "a\n\xF0\x90\x8D\x88\nz\n" 4 lines
+		// Last line empty
+		cb.InsertString(strlen(data), "\n", 1, startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 5);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF16) == 7);
+		REQUIRE(cb.IndexLineStart(4, SC_LINECHARACTERINDEX_UTF16) == 7);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 4);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF32) == 6);
+		REQUIRE(cb.IndexLineStart(4, SC_LINECHARACTERINDEX_UTF32) == 6);
+
+		// Insert a new line before end -> "a\n\xF0\x90\x8D\x88\nz\n\n" 5 lines
+		cb.InsertString(strlen(data), "\n", 1, startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 5);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF16) == 7);
+		REQUIRE(cb.IndexLineStart(4, SC_LINECHARACTERINDEX_UTF16) == 8);
+		REQUIRE(cb.IndexLineStart(5, SC_LINECHARACTERINDEX_UTF16) == 8);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 4);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF32) == 6);
+		REQUIRE(cb.IndexLineStart(4, SC_LINECHARACTERINDEX_UTF32) == 7);
+		REQUIRE(cb.IndexLineStart(5, SC_LINECHARACTERINDEX_UTF32) == 7);
+
+		// Insert a valid 3-byte UTF-8 character at start -> 
+		// "\xE2\x82\xACa\n\xF0\x90\x8D\x88\nz\n\n" 5 lines
+
+		const char *euro = "\xE2\x82\xAC";
+		cb.InsertString(0, euro, strlen(euro), startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 6);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF16) == 8);
+		REQUIRE(cb.IndexLineStart(4, SC_LINECHARACTERINDEX_UTF16) == 9);
+		REQUIRE(cb.IndexLineStart(5, SC_LINECHARACTERINDEX_UTF16) == 9);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 5);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF32) == 7);
+		REQUIRE(cb.IndexLineStart(4, SC_LINECHARACTERINDEX_UTF32) == 8);
+		REQUIRE(cb.IndexLineStart(5, SC_LINECHARACTERINDEX_UTF32) == 8);
+
+		// Insert a lone lead byte implying a 3 byte character at start of line 2 -> 
+		// "\xE2\x82\xACa\n\EF\xF0\x90\x8D\x88\nz\n\n" 5 lines
+		// Should be treated as a single byte character
+
+		const char *lead = "\xEF";
+		cb.InsertString(5, lead, strlen(lead), startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 7);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF16) == 9);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 6);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF32) == 8);
+
+		// Insert an ASCII lead byte inside the 3-byte initial character ->
+		// "\xE2!\x82\xACa\n\EF\xF0\x90\x8D\x88\nz\n\n" 5 lines
+		// It should b treated as a single character and should cause the
+		// byte before and the 2 bytes after also be each treated as singles
+		// so 3 more characters on line 0.
+
+		const char *ascii = "!";
+		cb.InsertString(1, ascii, strlen(ascii), startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 6);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 10);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 6);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 9);
+
+		// Insert a NEL after the '!' to trigger the utf8 line end case ->
+		// "\xE2!\xC2\x85 \x82\xACa\n \EF\xF0\x90\x8D\x88\n z\n\n" 5 lines
+
+		const char *nel = "\xC2\x85";
+		cb.InsertString(2, nel, strlen(nel), startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 7);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF16) == 11);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 7);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF32) == 10);
+	}
+
+	SECTION("Delete Multiple lines") {
+		cb.SetUTF8Substance(true);
+		cb.AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16 | SC_LINECHARACTERINDEX_UTF32);
+
+		bool startSequence = false;
+		// 3 lines of text containing 8 bytes
+		const char *data = "a\n\xF0\x90\x8D\x88\nz\nc";
+		cb.InsertString(0, data, strlen(data), startSequence);
+
+		// Delete first 2 new lines -> "az\nc"
+		cb.DeleteChars(1, strlen(data) - 4, startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 4);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 4);
+	}
+
+	SECTION("Delete Complex") {
+		cb.SetUTF8Substance(true);
+		cb.AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16 | SC_LINECHARACTERINDEX_UTF32);
+
+		bool startSequence = false;
+		// 3 lines of text containing 8 bytes
+		const char *data = "a\n\xF0\x90\x8D\x88\nz";
+		cb.InsertString(0, data, strlen(data), startSequence);
+
+		// Delete lead byte from character on line 1 ->
+		// "a\n\x90\x8D\x88\nz"
+		// line 1 becomes 4 single byte characters
+		cb.DeleteChars(2, 1, startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 6);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF16) == 7);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 6);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF32) == 7);
+
+		// Delete first new line ->
+		// "a\x90\x8D\x88\nz"
+		// Only 2 lines with line 0 containing 5 single byte characters
+		cb.DeleteChars(1, 1, startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 5);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 6);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 5);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 6);
+
+		// Restore lead byte from character on line 0 making a 4-byte character ->
+		// "a\xF0\x90\x8D\x88\nz"
+
+		const char *lead4 = "\xF0";
+		cb.InsertString(1, lead4, strlen(lead4), startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 4);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 5);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF32) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF32) == 3);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF32) == 4);
+	}
+
+	SECTION("Insert separates new line bytes") {
+		cb.SetUTF8Substance(true);
+		cb.AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF16 | SC_LINECHARACTERINDEX_UTF32);
+
+		bool startSequence = false;
+		// 2 lines of text containing 4 bytes
+		const char *data = "a\r\nb";
+		cb.InsertString(0, data, strlen(data), startSequence);
+
+		// 3 lines of text containing 5 bytes ->
+		// "a\r!\nb"
+		const char *ascii = "!";
+		cb.InsertString(2, ascii, strlen(ascii), startSequence);
+
+		REQUIRE(cb.IndexLineStart(0, SC_LINECHARACTERINDEX_UTF16) == 0);
+		REQUIRE(cb.IndexLineStart(1, SC_LINECHARACTERINDEX_UTF16) == 2);
+		REQUIRE(cb.IndexLineStart(2, SC_LINECHARACTERINDEX_UTF16) == 4);
+		REQUIRE(cb.IndexLineStart(3, SC_LINECHARACTERINDEX_UTF16) == 5);
+	}
+}
