Patch 8.1.2027

[Bram Moolenaar]

Patch 8.1.2027
Problem: MS-Windows: problem with ambiwidth characters.
Solution: handle ambiguous width characters in ConPTY on Windows 10 (1903).
(Nobuhiro Takasaki, closes #4411)
Files: src/Make_mvc.mak, src/Make_cyg_ming.mak, src/libvterm/src/parser.c,
src/libvterm/src/state.c, src/libvterm/src/termscreen.c,
src/libvterm/src/unicode.c, src/libvterm/src/vterm_internal.h,
src/misc2.c, src/os_win32.c, src/proto/misc2.pro,
src/proto/os_win32.pro


*** ../vim-8.1.2026/src/Make_mvc.mak 2019-09-12 22:26:19.991830017 +0200
--- src/Make_mvc.mak 2019-09-13 22:19:58.262938216 +0200
***************
*** 1716,1721 ****
--- 1716,1722 ----
-DVSNPRINTF=vim_vsnprintf \
-DIS_COMBINING_FUNCTION=utf_iscomposing_uint \
-DWCWIDTH_FUNCTION=utf_uint2cells \
+ -DGET_SPECIAL_PTY_TYPE_FUNCTION=get_special_pty_type \
-D_CRT_SECURE_NO_WARNINGS

# Create a default rule for libvterm.
*** ../vim-8.1.2026/src/Make_cyg_ming.mak 2019-09-07 23:16:16.826370964 +0200
--- src/Make_cyg_ming.mak 2019-09-13 22:29:38.984632426 +0200
***************
*** 1192,1198 ****
CCCTERM = $(CC) -c $(CFLAGS) -Ilibvterm/include -DINLINE="" \
-DVSNPRINTF=vim_vsnprintf \
-DIS_COMBINING_FUNCTION=utf_iscomposing_uint \
! -DWCWIDTH_FUNCTION=utf_uint2cells

$(OUTDIR)/%.o : libvterm/src/%.c $(TERM_DEPS)
$(CCCTERM) $< -o $@
--- 1192,1199 ----
CCCTERM = $(CC) -c $(CFLAGS) -Ilibvterm/include -DINLINE="" \
-DVSNPRINTF=vim_vsnprintf \
-DIS_COMBINING_FUNCTION=utf_iscomposing_uint \
! -DWCWIDTH_FUNCTION=utf_uint2cells \
! -DGET_SPECIAL_PTY_TYPE_FUNCTION=get_special_pty_type

$(OUTDIR)/%.o : libvterm/src/%.c $(TERM_DEPS)
$(CCCTERM) $< -o $@
*** ../vim-8.1.2026/src/libvterm/src/parser.c 2019-08-18 20:41:10.692526067 +0200
--- src/libvterm/src/parser.c 2019-09-13 22:24:15.133886984 +0200
***************
*** 127,132 ****
--- 127,135 ----
size_t pos = 0;
const char *string_start = NULL; // init to avoid gcc warning

+ vt->in_backspace = 0; // Count down with BS key and activate when
+ // it reaches 1
+
switch(vt->parser.state) {
case NORMAL:
case CSI_LEADER:
***************
*** 172,177 ****
--- 175,187 ----
// fallthrough
}
else if(c < 0x20) { // other C0
+ if(vterm_get_special_pty_type() == 2) {
+ if(c == 0x08) // BS
+ // Set the trick for BS output after a sequence, to delay backspace
+ // activation
+ if(pos + 2 < len && bytes[pos + 1] == 0x20 && bytes[pos + 2] == 0x08)
+ vt->in_backspace = 2; // Trigger when count down to 1
+ }
if(vt->parser.state >= STRING)
more_string(vt, string_start, bytes + pos - string_start);
do_control(vt, c);
*** ../vim-8.1.2026/src/libvterm/src/state.c 2019-08-18 20:41:10.692526067 +0200
--- src/libvterm/src/state.c 2019-09-13 22:19:58.262938216 +0200
***************
*** 336,341 ****
--- 336,346 ----

for( ; i < glyph_ends; i++) {
int this_width;
+ if(vterm_get_special_pty_type() == 2) {
+ state->vt->in_backspace -= (state->vt->in_backspace > 0) ? 1 : 0;
+ if(state->vt->in_backspace == 1)
+ codepoints[i] = 0; // codepoints under this condition must be 0
+ }
chars[i - glyph_starts] = codepoints[i];
this_width = vterm_unicode_width(codepoints[i]);
#ifdef DEBUG
***************
*** 425,430 ****
--- 430,441 ----

VTermPos oldpos = state->pos;

+ VTermScreenCell cell;
+
+ // Preparing to see the leading byte
+ VTermPos leadpos = state->pos;
+ leadpos.col -= (leadpos.col >= 2 ? 2 : 0);
+
switch(control) {
case 0x07: // BEL - ECMA-48 8.3.3
if(state->callbacks && state->callbacks->bell)
***************
*** 434,439 ****
--- 445,456 ----
case 0x08: // BS - ECMA-48 8.3.5
if(state->pos.col > 0)
state->pos.col--;
+ if(vterm_get_special_pty_type() == 2) {
+ // In 2 cell letters, go back 2 cells
+ vterm_screen_get_cell(state->vt->screen, leadpos, &cell);
+ if(vterm_unicode_width(cell.chars[0]) == 2)
+ state->pos.col--;
+ }
break;

case 0x09: // HT - ECMA-48 8.3.60
***************
*** 1019,1024 ****
--- 1036,1061 ----
row = CSI_ARG_OR(args[0], 1);
col = argcount < 2 || CSI_ARG_IS_MISSING(args[1]) ? 1 : CSI_ARG(args[1]);
// zero-based
+ if(vterm_get_special_pty_type() == 2) {
+ // Fix a sequence that is not correct right now
+ if(state->pos.row == row - 1) {
+ int cnt, ptr = 0;
+ for(cnt = 0; cnt < col - 1; ++cnt) {
+ VTermPos p;
+ VTermScreenCell c0, c1;
+ p.row = row - 1;
+ p.col = ptr;
+ vterm_screen_get_cell(state->vt->screen, p, &c0);
+ p.col++;
+ vterm_screen_get_cell(state->vt->screen, p, &c1);
+ ptr += (c1.chars[0] == (uint32_t)-1) // double cell?
+ ? (vterm_unicode_is_ambiguous(c0.chars[0])) // is ambiguous?
+ ? vterm_unicode_width(0x00a1) : 1 // &ambiwidth
+ : 1; // not ambiguous
+ }
+ col = ptr + 1;
+ }
+ }
state->pos.row = row-1;
state->pos.col = col-1;
if(state->mode.origin) {
*** ../vim-8.1.2026/src/libvterm/src/termscreen.c 2019-08-18 20:41:10.692526067 +0200
--- src/libvterm/src/termscreen.c 2019-09-13 22:19:58.262938216 +0200
***************
*** 770,780 ****
cell->fg = intcell->pen.fg;
cell->bg = intcell->pen.bg;

! if(pos.col < (screen->cols - 1) &&
! getcell(screen, pos.row, pos.col + 1)->chars[0] == (uint32_t)-1)
! cell->width = 2;
! else
! cell->width = 1;

return 1;
}
--- 770,797 ----
cell->fg = intcell->pen.fg;
cell->bg = intcell->pen.bg;

! if(vterm_get_special_pty_type() == 2) {
! /* Get correct cell width from cell information contained in line buffer */
! if(pos.col < (screen->cols - 1) &&
! getcell(screen, pos.row, pos.col + 1)->chars[0] == (uint32_t)-1) {
! if(getcell(screen, pos.row, pos.col)->chars[0] == 0x20) {
! getcell(screen, pos.row, pos.col)->chars[0] = 0;
! cell->width = 2;
! } else if(getcell(screen, pos.row, pos.col)->chars[0] == 0) {
! getcell(screen, pos.row, pos.col + 1)->chars[0] = 0;
! cell->width = 1;
! } else {
! cell->width = 2;
! }
! } else
! cell->width = 1;
! } else {
! if(pos.col < (screen->cols - 1) &&
! getcell(screen, pos.row, pos.col + 1)->chars[0] == (uint32_t)-1)
! cell->width = 2;
! else
! cell->width = 1;
! }

return 1;
}
*** ../vim-8.1.2026/src/libvterm/src/unicode.c 2019-08-18 20:41:10.692526067 +0200
--- src/libvterm/src/unicode.c 2019-09-13 22:27:24.625145881 +0200
***************
*** 68,79 ****
* Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
*/

- #if !defined(IS_COMBINING_FUNCTION) || !defined(WCWIDTH_FUNCTION)
struct interval {
int first;
int last;
};

// sorted list of non-overlapping intervals of non-spacing characters
// generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c"
// Replaced by the combining table from Vim.
--- 68,80 ----
* Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
*/

struct interval {
int first;
int last;
};

+ #if !defined(WCWIDTH_FUNCTION) || !defined(IS_COMBINING_FUNCTION)
+
// sorted list of non-overlapping intervals of non-spacing characters
// generated by "uniset +cat=Me +cat=Mn +cat=Cf -00AD +1160-11FF +200B c"
// Replaced by the combining table from Vim.
***************
*** 359,364 ****
--- 360,366 ----
{0X1E944, 0X1E94A},
{0XE0100, 0XE01EF}
};
+ #endif

// auxiliary function for binary search in interval table
static int bisearch(uint32_t ucs, const struct interval *table, int max) {
***************
*** 379,386 ****

return 0;
}
- #endif
-

/* The following two functions define the column width of an ISO 10646
* character as follows:
--- 381,386 ----
***************
*** 478,483 ****
--- 478,484 ----
*/
static int mk_wcwidth_cjk(uint32_t ucs)
{
+ #endif
/* sorted list of non-overlapping intervals of East Asian Ambiguous
* characters, generated by "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */
static const struct interval ambiguous[] = {
***************
*** 534,539 ****
--- 535,541 ----
{ 0x273D, 0x273D }, { 0x2776, 0x277F }, { 0xE000, 0xF8FF },
{ 0xFFFD, 0xFFFD }, { 0xF0000, 0xFFFFD }, { 0x100000, 0x10FFFD }
};
+ #if 0

// binary search in table of non-spacing characters
if (bisearch(ucs, ambiguous,
***************
*** 557,562 ****
--- 559,570 ----
}
#endif

+ INTERNAL int vterm_unicode_is_ambiguous(uint32_t codepoint)
+ {
+ return (bisearch(codepoint, ambiguous,
+ sizeof(ambiguous) / sizeof(struct interval) - 1)) ? 1 : 0;
+ }
+
#ifdef IS_COMBINING_FUNCTION
// Use a provided is_combining() function.
int IS_COMBINING_FUNCTION(uint32_t codepoint);
***************
*** 569,574 ****
--- 577,593 ----
}
#endif

+ #ifdef GET_SPECIAL_PTY_TYPE_FUNCTION
+ int GET_SPECIAL_PTY_TYPE_FUNCTION(void);
+ #else
+ # define GET_SPECIAL_PTY_TYPE_FUNCTION vterm_get_special_pty_type_placeholder
+ static int
+ vterm_get_special_pty_type_placeholder(void)
+ {
+ return 0;
+ }
+ #endif
+
// ################################
// ### The rest added by Paul Evans

***************
*** 581,583 ****
--- 600,607 ----
{
return IS_COMBINING_FUNCTION(codepoint);
}
+
+ INTERNAL int vterm_get_special_pty_type(void)
+ {
+ return GET_SPECIAL_PTY_TYPE_FUNCTION();
+ }
*** ../vim-8.1.2026/src/libvterm/src/vterm_internal.h 2019-08-18 20:41:10.692526067 +0200
--- src/libvterm/src/vterm_internal.h 2019-09-13 22:19:58.262938216 +0200
***************
*** 212,217 ****
--- 212,219 ----

VTermState *state;
VTermScreen *screen;
+
+ int in_backspace;
};

struct VTermEncoding {
***************
*** 259,263 ****
--- 261,267 ----

int vterm_unicode_width(uint32_t codepoint);
int vterm_unicode_is_combining(uint32_t codepoint);
+ int vterm_unicode_is_ambiguous(uint32_t codepoint);
+ int vterm_get_special_pty_type(void);

#endif
*** ../vim-8.1.2026/src/misc2.c 2019-09-04 20:59:10.487410001 +0200
--- src/misc2.c 2019-09-13 22:19:58.262938216 +0200
***************
*** 4601,4603 ****
--- 4601,4622 ----
}
# endif
#endif
+
+ /*
+ * Change the behavior of vterm.
+ * 0: As usual.
+ * 1: Windows 10 version 1809
+ * The bug causes unstable handling of ambiguous width character.
+ * 2: Windows 10 version 1903
+ * Use the wrong result because each result is different.
+ * 3: Windows 10 insider preview (current latest logic)
+ */
+ int
+ get_special_pty_type(void)
+ {
+ #ifdef MSWIN
+ return get_conpty_type();
+ #else
+ return 0;
+ #endif
+ }
*** ../vim-8.1.2026/src/os_win32.c 2019-07-31 20:53:52.545182527 +0200
--- src/os_win32.c 2019-09-13 22:19:58.266938202 +0200
***************
*** 186,191 ****
--- 186,192 ----
static int win32_set_archive(char_u *name);

static int conpty_working = 0;
+ static int conpty_type = 0;
static int conpty_stable = 0;
static void vtp_flag_init();

***************
*** 7249,7257 ****

/*
* Support for pseudo-console (ConPTY) was added in windows 10
! * version 1809 (October 2018 update). However, that version is unstable.
*/
#define CONPTY_FIRST_SUPPORT_BUILD MAKE_VER(10, 0, 17763)
#define CONPTY_STABLE_BUILD MAKE_VER(10, 0, 32767) // T.B.D.

static void
--- 7250,7274 ----

/*
* Support for pseudo-console (ConPTY) was added in windows 10
! * version 1809 (October 2018 update).
*/
#define CONPTY_FIRST_SUPPORT_BUILD MAKE_VER(10, 0, 17763)
+
+ /*
+ * ConPTY differences between versions, need different logic.
+ * version 1903 (May 2019 update).
+ */
+ #define CONPTY_1903_BUILD MAKE_VER(10, 0, 18362)
+
+ /*
+ * Confirm until this version. Also the logic changes.
+ * insider preview.
+ */
+ #define CONPTY_INSIDER_BUILD MAKE_VER(10, 0, 18898)
+
+ /*
+ * Not stable now.
+ */
#define CONPTY_STABLE_BUILD MAKE_VER(10, 0, 32767) // T.B.D.

static void
***************
*** 7281,7286 ****
--- 7298,7309 ----
if (ver >= CONPTY_STABLE_BUILD)
conpty_stable = 1;

+ if (ver <= CONPTY_INSIDER_BUILD)
+ conpty_type = 3;
+ if (ver <= CONPTY_1903_BUILD)
+ conpty_type = 2;
+ if (ver < CONPTY_FIRST_SUPPORT_BUILD)
+ conpty_type = 1;
}

#if !defined(FEAT_GUI_MSWIN) || defined(VIMDLL) || defined(PROTO)
***************
*** 7503,7508 ****
--- 7526,7537 ----
}

int
+ get_conpty_type(void)
+ {
+ return conpty_type;
+ }
+
+ int
is_conpty_stable(void)
{
return conpty_stable;
*** ../vim-8.1.2026/src/proto/misc2.pro 2019-09-04 20:59:10.491409987 +0200
--- src/proto/misc2.pro 2019-09-13 22:19:58.266938202 +0200
***************
*** 106,109 ****
--- 106,110 ----
int mch_parse_cmd(char_u *cmd, int use_shcf, char ***argv, int *argc);
int build_argv_from_string(char_u *cmd, char ***argv, int *argc);
int build_argv_from_list(list_T *l, char ***argv, int *argc);
+ int get_special_pty_type(void);
/* vim: set ft=c : */
*** ../vim-8.1.2026/src/proto/os_win32.pro 2019-04-28 22:50:36.157248454 +0200
--- src/proto/os_win32.pro 2019-09-13 22:19:58.266938202 +0200
***************
*** 76,81 ****
--- 76,82 ----
int is_term_win32(void);
int has_vtp_working(void);
int has_conpty_working(void);
+ int get_conpty_type(void);
int is_conpty_stable(void);
void resize_console_buf(void);
/* vim: set ft=c : */
*** ../vim-8.1.2026/src/version.c 2019-09-13 22:16:16.867909920 +0200
--- src/version.c 2019-09-13 22:28:01.729003238 +0200
***************
*** 759,760 ****
--- 759,762 ----
{ /* Add new patch number below this line */
+ /**/
+ 2027,
/**/

--
Emacs is a nice OS - but it lacks a good text editor.
That's why I am using Vim. --Anonymous

/// Bram Moolenaar -- Br...@Moolenaar.net -- http://www.Moolenaar.net \\\
/// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
\\\ an exciting new programming language -- http://www.Zimbu.org ///
\\\ help me help AIDS victims -- http://ICCF-Holland.org ///