next-3.1 consuming 2x memory

[Suraj N. Kurapati]

Hello,

After launching tmux and restoring my saved session, I observe these
memory usage numbers across the last, current, and future releases.
(There's only one instance of tmux running here: no double-counting.)

In particular, the VSZ has doubled in next-3.1 compared to 3.0-rc3.

tmux 2.9a
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
sunny 1667 0.0 0.0 7068 3292 pts/1 S+ 10:45 0:00 tmux
sunny 1669 18.1 9.2 374944 371184 ? Ss 10:45 0:06 tmux

tmux 3.0-rc3 (5a501a8a)
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
sunny 18479 0.0 0.0 7120 3256 pts/1 S+ 10:51 0:00 tmux
sunny 18481 14.7 9.2 375636 371864 ? Ss 10:51 0:05 tmux

tmux next-3.1 (68c2fc68)
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
sunny 27512 0.0 0.0 7124 3492 pts/1 S+ 10:52 0:00 tmux
sunny 27514 19.7 18.8 758776 755156 ? Ss 10:52 0:06 tmux

Thanks for your consideration.

[Nicholas Marriott]

Seems OK to me. How many windows, panes etc do you have?

> --
> You received this message because you are subscribed to the Google Groups "tmux-users" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to tmux-users+...@googlegroups.com.
> To post to this group, send an email to tmux-...@googlegroups.com.
> To view this discussion on the web, visit https://groups.google.com/d/msgid/tmux-users/20190628110739.1d132f6f%40ratham.attlocal.net.
> For more options, visit https://groups.google.com/d/optout.

[Suraj N. Kurapati]

I have a total of 9 windows and 32 panes in my saved session:

(0) - 0: 9 windows (attached)
(1) ├─> + 1: a* (6 panes)
(2) ├─> + 2: b- (8 panes)
(3) ├─> 3: c (1 panes)
(4) ├─> + 4: d (4 panes)
(5) ├─> + 5: e (3 panes)
(6) ├─> + 6: f (3 panes)
(7) ├─> + 7: g (2 panes)
(8) ├─> + 8: h (4 panes)
(9) └─> 9: i (1 panes)

[Nicholas Marriott]

I can't reproduce this with 32 panes. What does this show in each version:

tmux lsp -aF '#{pane_id} #{history_bytes}'

[Suraj N. Kurapati]

Great point! Indeed, I keep the maximum allowed scrollback histories:

set-option -g history-limit 32767

Here is the result of running the diagnostic command you had provided:

*** 3.0-rc ***

$ git rev-parse --short HEAD
5a501a8a

$ tmux -V
tmux 3.0-rc3

$ pgrep tmux | xargs -r ps u

USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND

sunny 5047 0.0 0.0 7120 3208 pts/1 S+ 09:16 0:00 tmux
sunny 5049 14.2 7.7 316064 311552 ? Rs 09:16 0:06 tmux

$ tmux lsp -aF '#{pane_id} #{history_bytes}'
%1 10414942
%2 11566347
%3 29081
%4 29299
%5 14790000
%6 19912
%7 19570538
%8 16828680
%9 45909
%10 708177
%11 1416667
%12 457673
%13 528808
%14 1296060
%15 9743781
%16 9868770
%17 7045780
%18 11011243
%19 835906
%20 22567043
%21 13197272
%22 15372503
%23 19222647
%24 12896611
%25 4660126
%26 11658149
%27 4434778
%28 11079426
%29 2222177
%30 124916
%31 302090
%32 194485

*** MASTER ***

$ git rev-parse --short HEAD
be5af704

$ tmux -V
tmux next-3.1

$ pgrep tmux | xargs ps u

USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND

sunny 27524 0.0 0.0 7124 812 pts/1 S+ 09:08 0:00 tmux
sunny 27526 13.7 28.3 1141996 1135832 ? Rs 09:08 0:13 tmux

$ tmux lsp -aF '#{pane_id} #{history_bytes}'
%1 13513750
%2 21892475
%3 147521
%4 137819
%5 124611630
%6 46864
%7 40913858
%8 114684856
%9 298109
%10 1429057
%11 9660307
%12 3092193
%13 3463888
%14 1382540
%15 23361921
%16 14491614
%17 52734235
%18 94974603
%19 879626
%20 28649231
%21 103158320
%22 127392471
%23 54594903
%24 105133611
%25 37563986
%26 37268993
%27 12498662
%28 15138678
%29 18205593
%30 1078476
%31 1221090
%32 1609525

On Mon, 1 Jul 2019 09:08:56 +0100, Nicholas Marriott wrote:
> I can't reproduce this with 32 panes. What does this show in each
> version:
>
> tmux lsp -aF '#{pane_id} #{history_bytes}'
>
> On Fri, Jun 28, 2019 at 12:43:59PM -0700, Suraj N. Kurapati wrote:
> > I have a total of 9 windows and 32 panes in my saved session:
> >

[Nicholas Marriott]

Yes the size of struct grid_cell increased to add the underscore colour,
you must have a lot of extended cells.

It does seem a bit weird because it was 36 bytes and is now 40 for me,
which is not much of an increase, perhaps your compiler is packing it
very badly.

But we can pack it better, please try this and see if it helps:

Index: grid.c
===================================================================
RCS file: /cvs/src/usr.bin/tmux/grid.c,v
retrieving revision 1.96
diff -u -p -r1.96 grid.c
--- grid.c 27 Jun 2019 15:17:41 -0000 1.96
+++ grid.c 3 Jul 2019 16:49:45 -0000
@@ -37,12 +37,12 @@

/* Default grid cell data. */
const struct grid_cell grid_default_cell = {
- 0, 0, 8, 8, 0, { { ' ' }, 0, 1, 1 }
+ { { ' ' }, 0, 1, 1 }, 0, 0, 8, 8, 0
};

/* Cleared grid cell data. */
const struct grid_cell grid_cleared_cell = {
- GRID_FLAG_CLEARED, 0, 8, 8, 0, { { ' ' }, 0, 1, 1 }
+ { { ' ' }, 0, 1, 1 }, 0, GRID_FLAG_CLEARED, 8, 8, 0
};
static const struct grid_cell_entry grid_cleared_entry = {
GRID_FLAG_CLEARED, { .data = { 0, 8, 8, ' ' } }
Index: screen-write.c
===================================================================
RCS file: /cvs/src/usr.bin/tmux/screen-write.c,v
retrieving revision 1.154
diff -u -p -r1.154 screen-write.c
--- screen-write.c 27 Jun 2019 15:17:41 -0000 1.154
+++ screen-write.c 3 Jul 2019 16:49:45 -0000
@@ -36,7 +36,7 @@ static const struct grid_cell *screen_wr
const struct utf8_data *, u_int *);

static const struct grid_cell screen_write_pad_cell = {
- GRID_FLAG_PADDING, 0, 8, 8, 0, { { 0 }, 0, 0, 0 }
+ { { 0 }, 0, 0, 0 }, 0, GRID_FLAG_PADDING, 0, 8, 8
};

struct screen_write_collect_item {
Index: style.c
===================================================================
RCS file: /cvs/src/usr.bin/tmux/style.c,v
retrieving revision 1.22
diff -u -p -r1.22 style.c
--- style.c 1 Jul 2019 06:56:00 -0000 1.22
+++ style.c 3 Jul 2019 16:49:45 -0000
@@ -30,7 +30,7 @@

/* Default style. */
static struct style style_default = {
- { 0, 0, 8, 8, 0, { { ' ' }, 0, 1, 1 } },
+ { { { ' ' }, 0, 1, 1 }, 0, 0, 8, 8, 0 },

8,
STYLE_ALIGN_DEFAULT,
Index: tmux.h
===================================================================
RCS file: /cvs/src/usr.bin/tmux/tmux.h,v
retrieving revision 1.918
diff -u -p -r1.918 tmux.h
--- tmux.h 1 Jul 2019 06:56:00 -0000 1.918
+++ tmux.h 3 Jul 2019 16:49:45 -0000
@@ -596,13 +596,13 @@ enum utf8_state {

/* Grid cell data. */
struct grid_cell {
- u_char flags;
+ struct utf8_data data; /* 21 bytes */
u_short attr;
+ u_char flags;
int fg;
int bg;
int us;
- struct utf8_data data;
-};
+} __packed;
struct grid_cell_entry {
u_char flags;
union {

[Suraj N. Kurapati]

Thanks for the patch! I've applied it atop master (at commit be5af704)
and re-tested as follows; unfortunately, it doesn't seem to be enough:

$ pgrep tmux | xargs -r ps u
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND

sunny 29809 0.0 0.0 7124 3328 pts/1 S+ 10:26 0:00 tmux
sunny 29811 15.7 26.1 1050788 1045744 ? Rs 10:26 0:11 tmux

$ tmux lsp -aF '#{pane_id} #{history_bytes}'

%1 12352246
%2 21097851
%3 0
%4 126967
%5 113399862
%6 44104
%7 37920818
%8 104899188
%9 272889
%10 1544601
%11 8835943
%12 2828741
%13 3170380
%14 1333320
%15 21239769
%16 13284234
%17 48164824
%18 86578267
%19 875254
%20 26066171
%21 94160444
%22 116190395
%23 49716951
%24 95909911
%25 34273150
%26 34015805
%27 12252574
%28 13825470
%29 16606409
%30 983120
%31 1129190
%32 1468021

I've performed a git bisection to find the culprit for you, as follows.
Along the way, I had to skip some commits because their Makefile was
tailored to BSD and refused to run `make install` on my Linux machine.

# possible first bad commit:
[dae2868d1227b95fd076fb4a5efa6256c7245943] Add support for underscore
colours with Setulc capability, mostly from Kai Moschcau.

$ git bisect bad
There are only 'skip'ped commits left to test.
The first bad commit could be any of:
dae2868d1227b95fd076fb4a5efa6256c7245943
e483ce138fbae32491df7e2dea23690e5aca3894
We cannot bisect more!
exit 2

$ git bisect log
git bisect start
# bad: [68c2fc682493f154d833ae6195b4adc33fd4a3b2] Merge branch 'obsd-master'
git bisect bad 68c2fc682493f154d833ae6195b4adc33fd4a3b2
# good: [5a501a8ae27c2d0128870caa48c5708e97528567] Pass keys that aren't 0-9 on to normal key processing when display-panes is active (restores previous behaviour).
git bisect good 5a501a8ae27c2d0128870caa48c5708e97528567
# good: [797042584e2a0b6208aafa3276a08588120a4ba4] Handle comments more correctly inside {}, from Avi Halachmi.
git bisect good 797042584e2a0b6208aafa3276a08588120a4ba4
# good: [4bbf941436765201825c8d675f45b0cb70eb19d3] Merge branch '3.0-rc'
git bisect good 4bbf941436765201825c8d675f45b0cb70eb19d3
# skip: [4ff7bc3eb32e9d66312d16757fb8c083df2d87d6] When system calls indicate an error they return -1, not some arbitrary value < 0. errno is only updated in this case. Change all (most?) callers of syscalls to follow this better, and let's see if this strictness helps us in the future.
git bisect skip 4ff7bc3eb32e9d66312d16757fb8c083df2d87d6
# good: [df07723e2093762d1cffc1a3f542e20853c42101] Add a config.
git bisect good df07723e2093762d1cffc1a3f542e20853c42101
# good: [69a2f73449a96228fb9b39b284fbc356e69eef73] Merge branch 'obsd-master'
git bisect good 69a2f73449a96228fb9b39b284fbc356e69eef73
# skip: [6ce38b73956836c26c8914cdf5002da7900a5d2d] asprintf returns -1, not an arbitrary value < 0. Also upon error the (very sloppy specification) leaves an undefined value in *ret, so it is wrong to inspect it, the error condition is enough. discussed a little with nicm, and then much more with millert until we were exasperated
git bisect skip 6ce38b73956836c26c8914cdf5002da7900a5d2d
# skip: [d83f356218fc9144735667e39c9553bcf905d10b] Add #define for the pane status line option position numbers.
git bisect skip d83f356218fc9144735667e39c9553bcf905d10b
# skip: [f797ac9ff6cd18b9f5737bea80fbf58dc5b5729b] Merge branch 'obsd-master'
git bisect skip f797ac9ff6cd18b9f5737bea80fbf58dc5b5729b
# skip: [c14b0d7c00373d08b7704a54c88a181aeed8f2ca] Merge branch 'obsd-master'
git bisect skip c14b0d7c00373d08b7704a54c88a181aeed8f2ca
# skip: [dae2868d1227b95fd076fb4a5efa6256c7245943] Add support for underscore colours with Setulc capability, mostly from Kai Moschcau.
git bisect skip dae2868d1227b95fd076fb4a5efa6256c7245943
# skip: [a07df21e79e9a6a1419f75bc45c4b9914b7efa92] Merge branch 'obsd-master'
git bisect skip a07df21e79e9a6a1419f75bc45c4b9914b7efa92
# skip: [b434692db20e6ef279ca13b6d25b0d519ac2c134] minor eol issues;
git bisect skip b434692db20e6ef279ca13b6d25b0d519ac2c134
# skip: [80d76612b8e8a572f96a58bcbd217f81d58d9b0f] Fix some comments (top/bottom not left/right).
git bisect skip 80d76612b8e8a572f96a58bcbd217f81d58d9b0f
# skip: [3a6d90adadfcd4aa6b513df7f8ae5c4dcc05a6dc] Fix a typo in window_pane_find_down (w not wp) and a missing PANE_STATUS_TOP.
git bisect skip 3a6d90adadfcd4aa6b513df7f8ae5c4dcc05a6dc
# skip: [20b938bcb18b8ae8b0535a1bcf8e7e1670a830bc] Expand arguments to C and s format modifiers (matches m which already expands).
git bisect skip 20b938bcb18b8ae8b0535a1bcf8e7e1670a830bc
# good: [c4a92999563e20617f949ac781f1465843fb088c] Merge branch 'obsd-master'
git bisect good c4a92999563e20617f949ac781f1465843fb088c
# bad: [b6b4f86cfc34f9386819af8f04add72833ccc69a] Merge branch 'obsd-master'
git bisect bad b6b4f86cfc34f9386819af8f04add72833ccc69a
# bad: [e483ce138fbae32491df7e2dea23690e5aca3894] Merge branch 'obsd-master'
git bisect bad e483ce138fbae32491df7e2dea23690e5aca3894
# only skipped commits left to test
# possible first bad commit: [e483ce138fbae32491df7e2dea23690e5aca3894] Merge branch 'obsd-master'
# possible first bad commit: [dae2868d1227b95fd076fb4a5efa6256c7245943] Add support for underscore colours with Setulc capability, mostly from Kai Moschcau.
# bad: [e483ce138fbae32491df7e2dea23690e5aca3894] Merge branch 'obsd-master'
git bisect bad e483ce138fbae32491df7e2dea23690e5aca3894
# only skipped commits left to test
# possible first bad commit: [e483ce138fbae32491df7e2dea23690e5aca3894] Merge branch 'obsd-master'
# possible first bad commit: [dae2868d1227b95fd076fb4a5efa6256c7245943] Add support for underscore colours with Setulc capability, mostly from Kai Moschcau.

[Nicholas Marriott]

Apply this please and show me the output of

tmux display -p '#{history_bytes}'

In any pane.


Index: format.c
===================================================================
RCS file: /cvs/src/usr.bin/tmux/format.c,v
retrieving revision 1.207
diff -u -p -r1.207 format.c
--- format.c 24 Jun 2019 10:04:29 -0000 1.207
+++ format.c 3 Jul 2019 19:12:18 -0000
@@ -612,7 +612,7 @@ format_cb_history_bytes(struct format_tr
}
size += gd->hsize * sizeof *gl;

- xasprintf(&fe->value, "%llu", size);
+ xasprintf(&fe->value, "%llu,%zu", size, sizeof (struct grid_cell));
}

/* Callback for pane_tabs. */

[Suraj N. Kurapati]

On Wed, 3 Jul 2019 20:12:45 +0100, Nicholas Marriott wrote:
> Apply this please and show me the output of
>
> tmux display -p '#{history_bytes}'
>
> In any pane.

With both patches applied, here is the new VSZ and diagnostic output:

$ pgrep tmux | xargs -r ps u
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND

sunny 2064 0.0 0.0 7124 3384 pts/1 S+ 14:07 0:00 tmux
sunny 2066 22.6 26.1 1050128 1045272 ? Rs 14:07 0:11 tmux

$ tmux display -p '#{history_bytes}'
113399862,36

$ tmux lsp -aF '#{pane_id} #{history_bytes}'

%1 12352246,36
%2 21097851,36
%3 135677,36
%4 126967,36
%5 113399862,36
%6 44104,36
%7 37920818,36
%8 104899188,36
%9 272889,36
%10 1544601,36
%11 8835943,36
%12 2828741,36
%13 3170380,36
%14 1333320,36
%15 21239769,36
%16 13284234,36
%17 48166564,36
%18 86578267,36
%19 875254,36
%20 26066171,36
%21 94160444,36
%22 116190395,36
%23 49716951,36
%24 95909911,36
%25 34273150,36
%26 34015805,36
%27 12252574,36
%28 13825470,36
%29 16606409,36
%30 983120,36
%31 1129190,36
%32 1468021,36

[Suraj N. Kurapati]

Also, for contrast, here is the effect of the same patch on 3.0-rc.
Both 3.0-rc (printf patch) and master (compaction + printf patches)
are reporting the same 36 number for the size of struct grid_cell.

$ pgrep tmux | xargs -r ps u
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND

sunny 11467 0.0 0.0 7120 3280 pts/1 S+ 14:13 0:00 tmux
sunny 11469 1.3 7.7 313312 309536 ? Rs 14:13 0:09 tmux

$ tmux display -p '#{history_bytes}'

68939,36

$ tmux lsp -aF '#{pane_id} #{history_bytes}'

%1 10414942,36
%2 11566347,36
%3 68939,36
%4 29299,36
%5 14790000,36
%6 19912,36
%7 19570538,36
%8 16828680,36
%9 45909,36
%10 708177,36
%11 1416667,36
%12 457673,36
%13 528808,36
%14 1296060,36
%15 9743781,36
%16 9868770,36
%17 7045780,36
%18 11011243,36
%19 835906,36
%20 22567043,36
%21 13197272,36
%22 15372503,36
%23 19222647,36
%24 12896611,36
%25 4660126,36
%26 11658149,36
%27 4434778,36
%28 11079426,36
%29 2222177,36
%30 124916,36
%31 302090,36
%32 194485,36

[Nicholas Marriott]

Does it fix it if you revert the underscore colour commit?

[Suraj N. Kurapati]

Yes, reverting commit dae2868 on top of master (at be5af70) fixes it:

$ pgrep tmux | xargs -r ps u
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND

sunny 8785 0.0 0.0 7124 3416 pts/1 S+ 20:24 0:00 tmux
sunny 8787 1.9 7.6 311848 308104 ? Rs 20:24 0:08 tmux

$ git log --oneline | head -3
ac5ca8a6 (HEAD -> revive) Revert "Add support for underscore colours with Setulc capability, mostly from"
be5af704 (origin/master, origin/HEAD) Merge branch 'obsd-master'
6a489fa7 Command prompt key presses need to avoid the command queue, GitHub issue 1817. Also a tmux.1 fix from jmc.

[Nicholas Marriott]

Try this please:

Index: format-draw.c
===================================================================
RCS file: /cvs/src/usr.bin/tmux/format-draw.c,v
retrieving revision 1.11
diff -u -p -r1.11 format-draw.c
--- format-draw.c 1 Jul 2019 06:56:00 -0000 1.11
+++ format-draw.c 5 Jul 2019 07:42:28 -0000
@@ -565,7 +565,7 @@ format_draw(struct screen_write_ctx *oct
cp++;
}

- /* Draw the cell to th current screen. */
+ /* Draw the cell to the current screen. */
screen_write_cell(&ctx[current], &sy.gc);
width[current] += ud->width;
continue;

Index: grid.c
===================================================================
RCS file: /cvs/src/usr.bin/tmux/grid.c,v
retrieving revision 1.96
diff -u -p -r1.96 grid.c
--- grid.c 27 Jun 2019 15:17:41 -0000 1.96

+++ grid.c 5 Jul 2019 07:42:28 -0000

@@ -37,12 +37,12 @@

/* Default grid cell data. */
const struct grid_cell grid_default_cell = {
- 0, 0, 8, 8, 0, { { ' ' }, 0, 1, 1 }
+ { { ' ' }, 0, 1, 1 }, 0, 0, 8, 8, 0
};

/* Cleared grid cell data. */
const struct grid_cell grid_cleared_cell = {
- GRID_FLAG_CLEARED, 0, 8, 8, 0, { { ' ' }, 0, 1, 1 }
+ { { ' ' }, 0, 1, 1 }, 0, GRID_FLAG_CLEARED, 8, 8, 0
};
static const struct grid_cell_entry grid_cleared_entry = {
GRID_FLAG_CLEARED, { .data = { 0, 8, 8, ' ' } }

@@ -475,6 +475,7 @@ grid_get_cell1(struct grid_line *gl, u_i
gc->bg = gce->data.bg;
if (gce->flags & GRID_FLAG_BG256)
gc->bg |= COLOUR_FLAG_256;
+ gc->us = 0;
utf8_set(&gc->data, gce->data.data);

}

Index: screen-write.c
===================================================================
RCS file: /cvs/src/usr.bin/tmux/screen-write.c,v
retrieving revision 1.154
diff -u -p -r1.154 screen-write.c
--- screen-write.c 27 Jun 2019 15:17:41 -0000 1.154

+++ screen-write.c 5 Jul 2019 07:42:28 -0000

@@ -36,7 +36,7 @@ static const struct grid_cell *screen_wr
const struct utf8_data *, u_int *);

static const struct grid_cell screen_write_pad_cell = {
- GRID_FLAG_PADDING, 0, 8, 8, 0, { { 0 }, 0, 0, 0 }
+ { { 0 }, 0, 0, 0 }, 0, GRID_FLAG_PADDING, 0, 8, 8
};

struct screen_write_collect_item {
Index: style.c
===================================================================
RCS file: /cvs/src/usr.bin/tmux/style.c,v
retrieving revision 1.22
diff -u -p -r1.22 style.c
--- style.c 1 Jul 2019 06:56:00 -0000 1.22

+++ style.c 5 Jul 2019 07:42:28 -0000

@@ -30,7 +30,7 @@

/* Default style. */
static struct style style_default = {
- { 0, 0, 8, 8, 0, { { ' ' }, 0, 1, 1 } },
+ { { { ' ' }, 0, 1, 1 }, 0, 0, 8, 8, 0 },

8,
STYLE_ALIGN_DEFAULT,
Index: tmux.h
===================================================================
RCS file: /cvs/src/usr.bin/tmux/tmux.h,v
retrieving revision 1.918
diff -u -p -r1.918 tmux.h
--- tmux.h 1 Jul 2019 06:56:00 -0000 1.918

+++ tmux.h 5 Jul 2019 07:42:28 -0000

@@ -596,13 +596,13 @@ enum utf8_state {

/* Grid cell data. */
struct grid_cell {
- u_char flags;
+ struct utf8_data data; /* 21 bytes */
u_short attr;
+ u_char flags;
int fg;
int bg;
int us;
- struct utf8_data data;
-};
+} __packed;
struct grid_cell_entry {
u_char flags;
union {

[Suraj N. Kurapati]

That did the trick! :) VSZ has reduced down to 3.0-rc levels, thanks!

$ pgrep tmux | xargs -r ps u
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND

sunny 350 0.0 0.0 7124 3420 pts/1 S+ 15:52 0:00 tmux
sunny 352 10.9 7.7 314640 310044 ? Rs 15:52 0:06 tmux

$ tmux lsp -aF '#{pane_id} #{history_bytes}'

%1 10414942
%2 11566347
%3 29081
%4 29299

%5 12493950

$ git log --oneline | head -3

aa085e68 underscore-2x-memory-try3.patch
bc112a8c Merge branch 'obsd-master'
55c694a4 Do not use uninitialized buffer name.