Debugging ydb termios calls -- What does PASTHRU actually do?

[Kevin Toppenberg]

Purpose: determine why USE $P:PASTHRU  doesn't put YDB into tty raw mode

YottaDB needs to tell the terminal system how to operate so regularly talks back and forth to it via two key calls:
tcsetattr()
tcgetattr()

Other calls are listed as part of the library, but they do not seem key to my investigation
       int tcgetattr(int fd, struct termios* termios_p);
       int tcsetattr(int fd, int optional_actions, const struct termios* termios_p);
       int tcsendbreak(int fd, int duration);
       int tcdrain(int fd);
       int tcflush(int fd, int queue_selector);
       int tcflow(int fd, int action);
       void cfmakeraw(struct termios* termios_p);
       speed_t cfgetispeed(const struct termios* termios_p);
       speed_t cfgetospeed(const struct termios* termios_p);
       int cfsetispeed(struct termios *termios_p, speed_t speed);
       int cfsetospeed(struct termios *termios_p, speed_t speed);
       int cfsetspeed(struct termios *termios_p, speed_t speed);

The key part of information passed into and out of the get/set attr calls is a pointer to struct termios

While debugging, if I dump out the termios record (stored in variable t), I see:
(gdb) print -pretty -- t
$9 = {
  c_iflag = 1280,
  c_oflag = 5,
  c_cflag = 191
  c_lflag = 35387,
  c_line = 0 '\000'
  c_cc =  ....
  c_ispeed = 15,
  c_ospeed = 15
}

Each of these flag variables is a bit record, where each bit specifies a different tty setting

I did a search through all the source code for all instances of tcsetattr or tcgetattr (and Tcsetattr, Tcgetattr which is a wrapper macro)

Here are hits for tcgetattr
    sr_unix/iott_open.c
    sr_unix/iott_use.c
    sr_unix/cli_lex.c
    sr_unix/mu_term_setup.c

and for tcsetattr
    sr_unix/iott_use.c
    sr_unix/setterm.c
    sr_unix/resetterm.c:
    sr_unix/mu_term_setup.c:

The key file for setting device parameters seems to be iott_use  (which Sam also previously pointed out).   The source code for this file is here.

I set breakpoints in gdb in the various routines above and the entered mumps command

USE $P:PASTHRU

This leads to breakpoint in iott_use.c
124:  status = tcgetattr(tt_ptr->fildes, &t);

this loads t with the current tty state.

I the asked gdb to display the some of elements of t as binary numbers:
display/t t.c_iflag
display/t t.c_cflag
display/t t.c_iflag

which gives this output:
/t t.c_iflag = 10100000000
/t t.c_cflag = 10111111
/t t.c_lflat = 1000101000111011

The meaning of each bit is defined, and I tracked them all down, but that doesn't matter here.

Next we come to  lined 361-2
361 case iop_pasthru:
362    mask_in |= TRM_PASTHRU;  //kt 000010000

So here we are doing some bit masking to set the parameter.  
It is storing this in mask_in, which is a simple int in local scope of the function.
mask_in is copied from a larger structure.  It will be put back later... more on this to come

The code then goes through the various settings, and sets flags.  Not all options result in flags set

Below is a simplified (edited) listing of flag elements changed:
-------------------------
case iop_canonical:    tt_ptr->canonical = TRUE;  
                       t.c_lflag |= ICANON;            //kt binary 00000010.
case iop_nocanonical:  tt_ptr->canonical = FALSE;
                       t.c_lflag &= ~(ICANON);         //kt binary 11111101.
case iop_empterm:      tt_ptr->ext_cap |= TT_EMPTERM;  //kt binary 0100 0000 0000 0000.
case iop_noempterm:    tt_ptr->ext_cap &= ~TT_EMPTERM; //kt binary 1011 1111 1111 11111
case iop_cenable:      (doesn't set flags)
case iop_nocenable:    (doesn't set flags)
case iop_clearscreen:  (doesn't set flags)
case iop_convert:      mask_in |= TRM_CONVERT;         //kt binary 100000000
case iop_noconvert:    mask_in &= ~TRM_CONVERT;        //kt binary 011111111
case iop_ctrap:        (doesn't set flags)
case iop_echo:         mask_in &= (~TRM_NOECHO);       //kt binary 101111111
case iop_noecho:       mask_in |= TRM_NOECHO;          //kt binary 010000000
case iop_editing:      tt_ptr->ext_cap |= TT_EDITING;
case iop_noediting:    tt_ptr->ext_cap &= ~TT_EDITING;
case iop_escape:       mask_in |= TRM_ESCAPE;          //kt binary 010000000
case iop_noescape:     mask_in &= (~TRM_ESCAPE);       //kt binary 101111111
case iop_eraseline:    (doesn't set flags)
case iop_exception:    (doesn't set flags)
case iop_filter:       (doesn't set flags)
  switch (fil_type) {
    case 0: iod->write_filter |= CHAR_FILTER;         //kt binary 10000000
    case 1: iod->write_filter |= ESC1;                //kt binary 00000001
    case 2: iod->write_filter &= ~CHAR_FILTER;        //kt binary 101111111
    case 3: iod->write_filter &= ~ESC1;               //kt binary 11111110
  }
case iop_nofilter:     iod->write_filter = 0;         //kt binary 00000000
case iop_flush:        (doesn't set flags)
case iop_hostsync:     t.c_iflag |= IXOFF;            //kt binary: 0001 0000 0000 0000
case iop_nohostsync:   t.c_iflag &= ~IXOFF;           //kt binary: 1110 1111 1111 1111
case iop_hupenable:    (doesn't set flags)
case iop_nohupenable:  (doesn't set flags)
case iop_insert:       tt_ptr->ext_cap &= ~TT_NOINSERT; //kt binary 1101 1111 1111 1111
case iop_noinsert:     tt_ptr->ext_cap |= TT_NOINSERT;  //kt binary 0010 0000 0000 0000
case iop_length:       (doesn't set flags)
case iop_pasthru:      mask_in |= TRM_PASTHRU;        //kt binary 000010000
case iop_nopasthru:    mask_in &= (~TRM_PASTHRU);     //kt binary 111101111
case iop_readsync:     mask_in |= TRM_READSYNC;       '/kt binary 000000100
case iop_noreadsync:   mask_in &= (~TRM_READSYNC);    //kt binary 111111011
case iop_terminator:   (doesn't set flags)
case iop_noterminator: (doesn't set flags)
case iop_ttsync:       t.c_iflag |= IXON;             //kt Binary: 0000 0100 0000 0000
case iop_nottsync:     t.c_iflag &= ~IXON;            //kt Binary: 1111 1011 1111 1111
case iop_typeahead:    mask_in &= (~TRM_NOTYPEAHD);   //kt 111111101
case iop_notypeahead:  mask_in |= TRM_NOTYPEAHD;      //kt 000000010
case iop_upscroll:     (doesn't set flags)
case iop_wrap:         (doesn't set flags)
case iop_nowrap:       (doesn't set flags)
case iop_x:            (doesn't set flags)
case iop_y:            (doesn't set flags)
case iop_ipchset:      (doesn't set flags)
case iop_opchset:      (doesn't set flags)
case iop_chset:        (doesn't set flags)

Below I am going to list them again, sorted by elements modified.
Items that modify tt_ptr-><element>
-------------------------
case iop_canonical:    tt_ptr->canonical = TRUE;  
case iop_nocanonical:  tt_ptr->canonical = FALSE;
case iop_empterm:      tt_ptr->ext_cap |= TT_EMPTERM;  //kt binary 0100 0000 0000 0000.
case iop_noempterm:    tt_ptr->ext_cap &= ~TT_EMPTERM; //kt binary 1011 1111 1111 11111
case iop_editing:      tt_ptr->ext_cap |= TT_EDITING;
case iop_noediting:    tt_ptr->ext_cap &= ~TT_EDITING;
case iop_insert:       tt_ptr->ext_cap &= ~TT_NOINSERT; //kt binary 1101 1111 1111 1111
case iop_noinsert:     tt_ptr->ext_cap |= TT_NOINSERT;  //kt binary 0010 0000 0000 0000


Items that directly modify t.<element>
case iop_canonical:    t.c_lflag |= ICANON;           //kt binary 00000010.
case iop_nocanonical:  t.c_lflag &= ~(ICANON);        //kt binary 11111101.
case iop_hostsync:     t.c_iflag |= IXOFF;            //kt binary: 0001 0000 0000 0000
case iop_nohostsync:   t.c_iflag &= ~IXOFF;           //kt binary: 1110 1111 1111 1111
case iop_ttsync:       t.c_iflag |= IXON;             //kt binary: 0000 0100 0000 0000
case iop_nottsync:     t.c_iflag &= ~IXON;            //kt binary: 1111 1011 1111 1111

Items that modify mask_in
-------------------------
case iop_convert:      mask_in |= TRM_CONVERT;        //kt binary 100000000
case iop_noecho:       mask_in |= TRM_NOECHO;         //kt binary 010000000
case iop_escape:       mask_in |= TRM_ESCAPE;         //kt binary 001000000
case iop_pasthru:      mask_in |= TRM_PASTHRU;        //kt binary 000010000
case iop_readsync:     mask_in |= TRM_READSYNC;       //kt binary 000000100
case iop_notypeahead:  mask_in |= TRM_NOTYPEAHD;      //kt binary 000000010

case iop_noconvert:    mask_in &= ~TRM_CONVERT;       //kt binary 011111111
case iop_echo:         mask_in &= (~TRM_NOECHO);      //kt binary 101111111
case iop_noescape:     mask_in &= (~TRM_ESCAPE);      //kt binary 110111111
case iop_nopasthru:    mask_in &= (~TRM_PASTHRU);     //kt binary 111101111
case iop_noreadsync:   mask_in &= (~TRM_READSYNC);    //kt binary 111111011
case iop_typeahead:    mask_in &= (~TRM_NOTYPEAHD);   //kt binary 111111101

Continuing with code:

  temp_ptr = (d_tt_struct *)d_in->dev_sp;
  Tcsetattr(tt_ptr->fildes, TCSANOW, &t, status, save_errno, CHANGE_TERM_TRUE);  
    /* Store both t.c_lflag and t.c_iflag back in tt_ptr after calling Tcsetattr */
    /* t.c_iflag used for show TTSYNC and NOTTSYNC when using ZSHOW "D" */
    /* we may not need t.c_lflag because tt_ptr->canonical is currently responsible for CANONICAL output */
    /* in sr_unix/zshow_devices.c but we might need it later so just store it both just in case */
    tt_ptr->ttio_struct->c_iflag = t.c_lflag;
    tt_ptr->ttio_struct->c_iflag = t.c_iflag;
    if (tt == d_in->type)    {
    temp_ptr->term_ctrl = mask_in;
    /* reset the mask to default if chset was changed without specifying new terminators or Default */
    }

Above we see that d_in->dev_sp->term_ctrl = mask_in  is the only way that canges to mask_in leave the function

**But note that the changes to mask_in are NOT written to tcsetattr**


I will repeat from above, then continue.  

USE $P:PASTHRU

This leads to breakpoint in iott_use.c
124:  status = tcgetattr(tt_ptr->fildes, &t);  <--- I break AFTER this line. Note this is state BEFORE any changes are made

this loads t with the current tty state.

I the asked gdb to display the some of elements of t as binary numbers:
display/t t.c_iflag
display/t t.c_cflag
display/t t.c_iflag

which gives this output:
/t t.c_iflag = 10100000000
/t t.c_cflag = 10111111
/t t.c_lflat = 1000101000111011

If I then type at the mumps prompt:

USE $P:PASTHRU

And again break at
124:  status = tcgetattr(tt_ptr->fildes, &t);  <--- I break AFTER this line. Note this is state BEFORE any changes are made

which gives this output:
/t t.c_iflag = 10100000000
/t t.c_cflag = 10111111
/t t.c_lflat = 1000101000111011

NOTICE that there is no change to the t structure which gets state from tty system.

Conclusion

PASTHRU / NOPASTHRU Does not changes any settings in the TTY system.  It must only change internal YDB behavior

Below are the only params that seem to change TTY settings

iop_canonical
iop_nocanonical  
iop_hostsync  
iop_nohostsync  
iop_ttsync    
iop_nottsync


Now I am confused, because i thought PASTHRU was going to change the TTY system.

KT

[Kevin Toppenberg]

Typo, the second setting of $P should be USE $P:NOPASTHRU

KT

[Kevin Toppenberg]

P.S.     There is also the cfmakeraw() in the termios library which is interesting.  But this linke here https://stackoverflow.com/questions/74170993/which-attribute-in-the-termios-struct-generated-by-cfmakeraw-is-responsible-fo discusses this, and it looks like it just is a wrapper for setting other fundamental flags. 

As an additional wrinkle, the above link mentions a VMIN and a VTIME as important variables when operating in non-canonical mode.

Regarding canonical / noncanonical, I previously didn't understand what this term in YDB meant.  But I realize now that it was reflect TTY terminology.  From the man page for termios:

> In  noncanonical  mode input is available immediately (without the user having to type a line-delimiter character), no input processing is per‐formed, and line editing is disabled.  The read buffer will only accept 4095 chars; this provides the necessary space for a newline char if the input  mode is switched to canonical.  The settings of MIN (c_cc[VMIN]) and TIME (c_cc[VTIME]) determine the circumstances in which  a  read(2) completes; there are four distinct cases:

I think it is the TTY line-editing functionality that is consuming the BACKSPACE char.  So my next investigations are going to focus on exploring NONCANONICAL YDB settings. 

I am also going to look more into the YDB function set_term().   When I used code "FOR I=1:1:5 READ *X W X,!"  the set_term() function is called for each iteration.  And this function calls tcsetattr().  So I plan to explore what settings are being sent to the TTY system at that time.

By the way, I am writing all this for my future reference.  Others are free to comment, but I am not asking specific questions here.  If I have a question, I will try to be clear about it.

Kevin

[Sam Habiel]

FYI: I took a quick look at the YDB Test system, and we test PASTHRU. So this means usually that it's working and does something useful somewhere. I don't know what exactly.

--Sam

--
You received this message because you are subscribed to the Google Groups "Everything MUMPS" group.
To unsubscribe from this group and stop receiving emails from it, send an email to everythingmum...@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/everythingmumps/d2d4f9f4-96f6-4414-82ee-8a6799c91d75n%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

[Kevin Toppenberg]

Well, I've spent a couple of days now kicking around the code.  It is quite complex.  I am spoiled by the debugging tools built into Delphi.  We have gotten so adept at tracking problems down in CPRS, that I thought I might have similar success here.  But this is hard for a variety of reasons!

QUESTION:

If I make a breakpoint on iott_use(), and then look at the call stack.  I see a call stack like this:

#0  iott_use

#1  op_use

#2 ??

#3 ??

#4 from /opt/yottadb/current/libyottadb.so   <--------- what is the nature of this?

#6 ??

#7 ... etc

What is libyottadb.so?  Why can't I step through this with source code?  Is it binary (assembly) only?

If I trace all until leaving iott_use, to back out of the call stack, eventually gdb complains about  "cannot find bounds of current function".

Is there a brief way to help me understand how the pieces go together?  E.g. is there a central thread, and various tasks are put into separate threads, and thus my back-tracing will never make it back to the mother thread?

Kevin

[Shane Dewitt]

In the old days, if you wanted to debug something with gdb, you had to compile with the -g flag to get extra debugging information. You could probably recompile the shared library libyottadb.so with debugging symbols.

To view this discussion visit https://groups.google.com/d/msgid/everythingmumps/76f55ad9-5d80-4171-8461-418c045c6d69n%40googlegroups.com.

[Kevin Toppenberg]

Shane,

I agree that compiling with debug symbols would be great.  But I have also heard that some core parts of yottadb are written in assembly.  This may be one of them.

Thanks for your feedback.

KT

[Sam Habiel]

Kevin,

You are right. I call this "The VAX Chasm". The bytecode for YottaDB is VAX assembly, and it's loaded into the C stack, and the previous stack is saved off in memory. So in GDB, you won't find any stack.

You DO NOT need the lower levels of the invocation. It's main -> gtm_init -> dm_mode -> t_something then it goes into the VAX compiled code, and from there it calls the C functions. So U $P goes into iott_use() right away.

--Sam

To view this discussion visit https://groups.google.com/d/msgid/everythingmumps/a961b796-587a-45db-a0d0-20710846c8f5n%40googlegroups.com.

[Kevin Toppenberg]

Thank you Sam and Shane for your feedback.

Sam, if it was just a matter of iott_use(), then I think I could solve the problem via use parameters (e.g. USE $P:(NOCANONICAL) etc).  Because from what I can tell iott_use is just taking the parameters and passing them on to the tty system.   And iott_use is only called when the the "USE" command is encountered.  It does call tcsetattr(), but it is NOT the only place that this is called.   If I put a breakpoint in there, and then enter "READ *X:.01 W:X>0 X,! HANG 0.1", then the breakpoint is never even hit. 

I think this is a subtle result of the interplay between yottadb and the tty system.  For example, I have noticed that during a loop, setterm.c:setterm() is called for each iteration.  Setterm() is another place that tcsetattr() is called to modify the tty system.  During each call a hardcoded 8 msec and character of 1 is also set.  I am wondering if this somehow resets the tty system input buffer such that it drops  127/backspace.  It is very difficult to debug because timing makes all the difference between backspace being consumed or not. 

Other tidbits I have discovered:  

• A different routine is call for READ X  vs READ *X    The latter is found in iott_rdone (which I think stands for 'READ ONE')
• in iott_rdone, before calling read() to get data from the io buffer, it first calls select(), which seems to be a standard c library call to tell if the io channel is ready.  If, by carefully stepping through the system, I can tell that when I enter BACKSPACE keys, select() is returning false, i.e. no data ready.  With normal system running, this causes the code to not even attempt a read().  If I manually override the return value to true, tricking it into thinking there is data waiting, and then go on to perform a read(), then I am able to enter a BACKSPACE, and it gets read as expected.   This would again point to the tty system not acting as expected.  If select() would return true, then the backspace would be read.  And as per other experiments, if the tty system doesn't have time to process its input (i.e. my loop is fast, without any HANG), then one will get through. 
  
• As per prior posts, entering linux command 'stty raw' will cause BACKSPACE reads to succeed.  This mode must cause the tty system to ignore calls to tcsetattr(), because program settings are being overwritten.  In the C library, there is a call cfmakeraw().  The documentation for this says:
         cfmakeraw() sets the terminal to something like the "raw" mode of the old Version 7 terminal driver: input is available character by character, echoing is disabled, and all special processing of terminal input and output characters is disabled.  The terminal attributes are set as follows:
  
             termios_p->c_iflag &= ~(IGNBRK | BRKINT | PARMRK | ISTRIP
                             | INLCR | IGNCR | ICRNL | IXON);
             termios_p->c_oflag &= ~OPOST;
             termios_p->c_lflag &= ~(ECHO | ECHONL | ICANON | ISIG | IEXTEN);
             termios_p->c_cflag &= ~(CSIZE | PARENB);
             termios_p->c_cflag |= CS8;
  
  I theoretically should be able to map these to USE:$P(xxxxx) settings and see if I can make this work, or perhaps do it during runtime debugging through manual mask overriding in gdb.

I am strongly tempted to give up on all this.  But I really want to be able to have a non-blocking message loop that can return a BACKSPACE key.  How can I write any dynamic programs if all READs are blocking?  And how can I get by without a BACKSPACE key?

I'm going to struggle on a bit longer.

KT 

[K.S. Bhaskar]

I am mostly out of my depth in this discussion. However, I wonder if there is a simpler solution: don't use YottaDB for terminal IO, but use another language, such as C, Python, Lua… The focus of YottaDB really is the database.

Regards

– Bhaskar

[Kevin Toppenberg]

Oh, how the might have fallen...

LOL!

Yes, I know that it would be much better to leave mumps with the relics of history, but some of us still live there.  Every day we are diving into mumps code etc.  I have written a mumps debugger that I am sort of proud of.  And I would like to make it better.... so here I am stuck trying to make a silk purse out of a ..., well you know what I mean .  :-)

I also thought about using the c extensions that yottadb seems to allow (I've not looked into it fully).  I.e. make an extension that brings in ncurses code.  But I thought that the IO channel would still belong to the mumps process, even if there is some c-code tacked onto the side.

Regarding depth of discussion... there is an interesting YouTube channel called CuriousMarc, where these guys resurrect communication hardware used in the space programs.  Its all about oscilloscopes detecting bits as the decode the wire protocol between the ground and the rocket.  I think the unix tty system was designed right around that same time, and they feel equally arcane. 

Thanks for not laughing me out the door for trying to look into it.  Ha!

:-)

Kevin

[Sam Habiel]

Kevin,

Try strace. It will show you what io calls are being made. strace -k will print the stack.

--Sam

To view this discussion visit https://groups.google.com/d/msgid/everythingmumps/43444dd5-e7c0-4a72-98dd-bdf2508a5c48n%40googlegroups.com.

[Kevin Toppenberg]

Sam, again an excellent suggestion.   I tried this before, but then forgot about it.  I'm going to post what I find, as I go along.

First setup.  yottadb is running inside the docker container.  the /data/dockertmp is a mapped folder to the host, so I can monitor output there without having to connect inside the container.

strace -o /data/dockertmp/debug_trace.txt -s 128 -v ./yottadb -dir  <-- -v to get full output of parameters, -s 128 for long screen lines.

In a separate terminal, I run this:

tail -f debug_trace.txt | grep -v -e "rt_sigprocmask(" -e "pselect6(" -e "timer_" -e "ioctl(0, TCGETS,"

I can then watch what is happening as I type. 

I typed, in yottadb, this: "read *X w X,!"  This WILL successfully read a backspace

• ioctl(0, SNDCTL_TMR_START or TCSETS, {c_iflags=0x500, c_oflags=0x5, c_cflags=0xbf, c_lflags=0x8a3b, c_line=0, c_cc="\x03\x1c\x7f\x15\x04\x00\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0
• brk(0x650e899b3000)                     = 0x650e899b3000
• openat(AT_FDCWD, "/dev/urandom", O_RDONLY) = 3
• newfstatat(3, "", {st_dev=makedev(0, 0x72), st_ino=10, st_mode=S_IFCHR|0666, st_nlink=1, st_uid=0, st_gid=0, st_blksize=4096, st_blocks=0, st_rdev=makedev(0x1, 0x9), st_atime=1733664872 /* 2024-12-08T13:34:32.639740576+0000 */, st_atime_nsec=639740576, st_mtime=1733664872 /* 2024-12-08T13:34:32.639740576+0000 */, st_mtime_nsec=639740576, st_ctime=1733664872 /* 2024-12-08T13:34:32.639740576+0000 */, st_ctime_nsec=639740576}, AT_EMPTY_PATH) = 0
• ioctl(3, TCGETS, 0x7ffd0c6a8780)        = -1 EINVAL (Invalid argument)
• read(3, "\224\263l9\fu\236v\315\304#\305x\222jx\222\322\307\22\t\360\364\310&\234{\377\301\324a\247\223\333\1\0`\372\366\264\364\205Mo.\314\t\3231\2'\262\305O\303\312f\241X\247\363\341\250\350\244\277\3029)\351T-W\367O\0\222\200\232k\362\342\353t\331\0342z`\4Rz\305\10\0324\t\353;\367\330\267\337\312\223\35\305\332\206\20R\0\364A\321%\2538^\307\32\273=Z\314\373\322\313"..., 4096) = 4096
• close(3)                                = 0
• write(0, "\n", 1)                       = 1
• ioctl(0, SNDCTL_TMR_START or TCSETS, {
    c_iflags=0x400, <-- binary 0100 0000 0000  <-- IXON (Enable start/stop output control)
    c_oflags=0x5,
    c_cflags=0xbf,
    c_lflags=0x8a31, <-- binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)
    c_line=0,
    c_cc[VMIN]=1,   <--- notice that this is here, more about this below
    c_cc[VTIME]=8, <--- notice that this is here, more about this below
    c_cc="\x03\x1c\x7f\x15\x04\x08\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0
• <---- HERE I TYPED BACKSPACE
• write(0, "127\n\n", 5)                  = 5    <---- here ydb is showing the "127" for the backspace
  
• write(0, "\r", 1)                       = 1
• write(0, "YDB>", 4)                     = 4
• write(0, "\33[?1h\33=", 7)              = 7

Next experiment will NOT be able to read backspace

I typed this in ydb:   for i=1:1:2 W "[" READ *X:0.1 W:X>0 X W "]" HANG 5

I got this output:

• ioctl(0, SNDCTL_TMR_START or TCSETS, {c_iflags=0x500, c_oflags=0x5, c_cflags=0xbf, c_lflags=0x8a3b, c_line=0, c_cc="\x03\x1c\x7f\x15\x04\x00\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0
• write(0, "\n[", 2)                      = 2
• ioctl(0, SNDCTL_TMR_START or TCSETS, {c_iflags=0x400, c_oflags=0x5, c_cflags=0xbf, c_lflags=0x8a31, c_line=0, c_cc[VMIN]=1, c_cc[VTIME]=8, c_cc="\x03\x1c\x7f\x15\x04\x08\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0
• ioctl(0, SNDCTL_TMR_START or TCSETS, {c_iflags=0x500, c_oflags=0x5, c_cflags=0xbf, c_lflags=0x8a3b, c_line=0, c_cc="\x03\x1c\x7f\x15\x04\x00\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0
• clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734051925, tv_nsec=715390561}, NULL) = ? ERESTARTNOHAND (To be restarted if no handler)
• --- SIGALRM {si_signo=SIGALRM, si_code=SI_TIMER, si_timerid=0x4, si_overrun=0, si_int=0, si_ptr=NULL} ---
• write(0, "]", 1)                        = 1
• <--- First read has completed
  
• rt_sigreturn({mask=[]})                 = -1 EINTR (Interrupted system call)
• <-- NOTE: This is the end of the first cycle.  It is so fast, I can't type anything in.  But now starting 5 second HANG, and I type many BACKSPACE keys during this time.
  
• clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734051925, tv_nsec=715390561}, NULL) = 0
• <--- hang 5 concluded
  
• write(0, "[", 1)                        = 1
• <--- second read starting
  
• ioctl(0, SNDCTL_TMR_START or TCSETS, {
  c_iflags=0x400,  <-- binary 0100 0000 0000  <-- IXON (Enable start/stop output control)
  c_oflags=0x5,
  c_cflags=0xbf,
  c_lflags=0x8a31,  <-- binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)
  c_line=0,
  c_cc[VMIN]=1,     <--- this is not set after overwritten below
  c_cc[VTIME]=8,   <--- this is not set after overwritten below
  c_cc="\x03\x1c\x7f\x15\x04\x08\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0
  
  
• Notice that above attribute set is OVERWRITTEN below!!!
  
  
• ioctl(0, SNDCTL_TMR_START or TCSETS, { 
  c_iflags=0x500,    <-- binary 0101 0000 0000  <-- IXON  (Enable start/stop output control) | ICRNL Map CR to NL on input. 
  c_oflags=0x5,  
  c_cflags=0xbf,
  c_lflags=0x8a3b,  <-- binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)
  c_line=0,
  c_cc="\x03\x1c\x7f\x15\x04\x00\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0
  
  
• clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734051930, tv_nsec=871214043}, NULL) = ? ERESTARTNOHAND (To be restarted if no handler)
• --- SIGALRM {si_signo=SIGALRM, si_code=SI_TIMER, si_timerid=0x5, si_overrun=0, si_int=0, si_ptr=NULL} ---
• write(0, "]", 1)                        = 1
• <--- second read has completed
  
• rt_sigreturn({mask=[]})                 = -1 EINTR (Interrupted system call)
• clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734051930, tv_nsec=871214043}, NULL) = 0
• <--- hang 5 concluded
  
• ioctl(0, SNDCTL_TMR_START or TCSETS, {c_iflags=0x400, c_oflags=0x5, c_cflags=0xbf, c_lflags=0x8a31, c_line=0, c_cc[VMIN]=1, c_cc[VTIME]=8, c_cc="\x03\x1c\x7f\x15\x04\x08\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0
• <-- notice: no 127 printed because backspace was not received
  
• write(0, "\n", 1)                       = 1
• write(0, "\r", 1)                       = 1
• write(0, "YDB>", 4)                     = 4
• write(0, "\33[?1h\33=", 7)              = 7
• read(0, "\177", 1)                      = 1

Conclusions:

• First experiment "read *X w X,!" IS ABLE to read BACKSPACE, 
• Second experiment "for i=1:1:2 W "[" READ *X:0.1 W:X>0 X W "]" HANG 5" is NOT able to read BACKSPACE
  
• First sets ICANON OFF, which turns OFF internal tty editing of line (i.e. does NOT use backspace key)
• Second sets ICANON OFF, but then turns it back ON.  I think this may be the culprit. 
  

So now I need to figure out where the 2nd attribute set is coming from.  

The plot thickens... :-)

Kevin

[Kevin Toppenberg]

OK, today I am going to investigate more why setterm() is being called twice.  1st time is appropriate setup, 2nd time overwrites the 1st, and sets CANONICAL mode which consumes backspace.

Ideally, I would be able to step through code with gdb and see how the steps cause output with strace.  However, it is not currently possible to run gdb and strace at the same time.  I found that strace is being modified to work with gdb server, but it is still experimental and on the RedHat platform.

I'm going to repeat part of prior post, but using different tests.  I will first document this in strace, then in gdb.

I'm using two lines of mumps code at tests. 

• HANG 10 READ *X  HANG 10
  
• HANG 10 READ *X:0.1  HANG 10
  

The idea behind the HANG's is that I can add markers in the output log, and distinguish between setup, the actual READ, and then restoration of IO for the prompt.

-----------------------------------

OK, first test: HANG 10 READ *X HANG 10

Output below with editing and formatting for readability

read(0, "\33", 1)                       = 1
read(0, "O", 1)                         = 1
read(0, "A", 1)                         = 1
read(0, "\r", 1)                        = 1

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x500, <-- binary 0101 0000 0000  IXON (Enable start/stop output ctrl) | ICRNL (Maps CR to NL on input)

c_oflags=0x5, 

c_cflags=0xbf, 

c_lflags=0x8a3b, <-- binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)

c_line=0, 

c_cc="\x03\x1c\x7f\x15\x04\x00\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734272633, tv_nsec=494337340}, NULL) = ? ERESTARTNOHAND (To be restarted if no handler)
--- SIGALRM {si_signo=SIGALRM, si_code=SI_TIMER, si_timerid=0x3, si_overrun=0, si_int=0, si_ptr=NULL} ---

rt_sigreturn({mask=[]})                 = -1 EINTR (Interrupted system call)

===== 1st HANG ========

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734272633, tv_nsec=494337340}, NULL) = 0

-- hang done --

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x400, <-- binary 0100 0000 0000  <-- IXON (Enable start/stop output control)

c_oflags=0x5, 

c_cflags=0xbf, 

c_lflags=0x8a31, <-- binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)

c_line=0, 

c_cc[VMIN]=1, 

c_cc[VTIME]=8, 

c_cc="\x03\x1c\x7f\x15\x04\x08\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0

read(0, "a", 1)                         = 1   <--- I pressed 'a' here

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x500, <-- binary 0101 0000 0000  <-- IXON  (Enable start/stop output control) | ICRNL Map CR to NL on input. 

c_oflags=0x5, 

c_cflags=0xbf, 

c_lflags=0x8a3b, <-- binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)

c_line=0, 

c_cc="\x03\x1c\x7f\x15\x04\x00\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0

===2nd HANG =========

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734272643, tv_nsec=496205068}, NULL) = 0

-- hang done --

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x400, <-- binary 0100 0000 0000  <-- IXON (Enable start/stop output control)

c_oflags=0x5, 

c_cflags=0xbf, 

c_lflags=0x8a31, <-- binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)

c_line=0, 

c_cc[VMIN]=1, 

c_cc[VTIME]=8, 

c_cc="\x03\x1c\x7f\x15\x04\x08\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0

--------------------------------------------------------------------------------

OK, now for 2nd test.  HANG 10 READ *X:0.1 HANG 10

tail -f debug_trace.txt | grep -v -e 'rt_sigprocmask(' -e 'pselect6(' -e 'timer_' -e 'ioctl(0, TCGETS,' -e 'write(0, "\\10", 1)' -e 'write(0, "\\33\[C", 3)' -e 'write(0, ' -e 'newfstatat' -e 'brk('

read(0, "\r", 1)                        = 1

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x500, <-- binary 0101 0000 0000  <-- IXON  (Enable start/stop output control) | ICRNL Map CR to NL on input. 

c_oflags=0x5, 

c_cflags=0xbf, 

c_lflags=0x8a3b, <-- binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)

c_line=0, 

c_cc="\x03\x1c\x7f\x15\x04\x00\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734271917, tv_nsec=332846273}, NULL) = ? ERESTARTNOHAND (To be restarted if no handler)
--- SIGALRM {si_signo=SIGALRM, si_code=SI_TIMER, si_timerid=0x3, si_overrun=0, si_int=0, si_ptr=NULL} ---

rt_sigreturn({mask=[]})                 = -1 EINTR (Interrupted system call)

=== 1st HANG ===

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734271917, tv_nsec=332846273}, NULL) = 0

-- hang done --

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x400, <-- binary 0100 0000 0000  <-- IXON (Enable start/stop output control)

c_oflags=0x5, 

c_cflags=0xbf, 

c_lflags=0x8a31, <-- binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)

c_line=0, 

c_cc[VMIN]=1,   <--- this is not set after overwritten below

c_cc[VTIME]=8, <--- this is not set after overwritten below

c_cc="\x03\x1c\x7f\x15\x04\x08\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x500, <-- binary 0101 0000 0000  <-- IXON  (Enable start/stop output control) | ICRNL Map CR to NL on input. 

c_oflags=0x5, 

c_cflags=0xbf, 

c_lflags=0x8a3b, <-- binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)

c_line=0, 

c_cc="\x03\x1c\x7f\x15\x04\x00\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0

===== 2nd HANG, after READ *X:0.1  ====

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734271927, tv_nsec=447731242}, NULL) = 0

-- hang done --

-- resetting IO in preparation for the YDB> prompt --

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x400, <-- binary 0100 0000 0000  <-- IXON (Enable start/stop output control)

c_oflags=0x5, 

c_cflags=0xbf, 

c_lflags=0x8a31, <-- binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)

c_line=0, 

c_cc[VMIN]=1, 

c_cc[VTIME]=8, 

c_cc="\x03\x1c\x7f\x15\x04\x08\x01\x00\x11\x13\x1a\x00\x12\x0f\x17\x16\x00\x00\x00"}) = 0

*****************************************************************************

OK, now for tracing in gdb with same tests.

*****************************************************************************

In gdb, I set only one breakpoint: setterm.c:setterm

I then enter HANG 10 READ *X HANG 10

After entering command, I immediately get break, with stack as follows:

• ?
• stkok3
• op_dmmode
• dm_read
• resetterm  <-- most recent/last call
  

After continuing, and waiting through HANG, next breakpoint shows stack as follows:

• ?
• op_rddone
• iott_rdone
• setterm <-- most recent/last call

after continuing, process waits for me to enter a key.  I enter a letter, and process then reaches breakpoint, showing stack as follows:

• ?
• op_rdone
• iott_rdone
• resetterm <-- most recent/last call

After continuing, and waiting through hang, next breakpoint shows stack:

• ?
• stkok3 in opp_dmmode.s
• op_dmode
• dm_read
• setterm <-- most recent/last call

after continuing, I am returned to YDB> prompt

------------------------

NEXT TEST:

HANG 10 READ *X:0.1 HANG 10

First beakpoint encounter shows stack as below.  This is BEFORE hang:

• ?
• stkok3 in opp_dmmode.s
• op_dmode
• dm_read
• setterm <-- most recent/last call

After continuing, and waiting through HANG, next breakpoint reached, showing stack:

• ?
• op_rdone
• iott_rdone
• setterm  <-- most recent/last call

After continuing, next breakpoint reached WITHOUT any delay (HANG), showing stack:

• ?
• op_rdone
• iott_rdone
• resetterm <-- most recent/last call

After continuing, and wanting through HANG, next breakpoint reached, showing stack:

• ?
• stkok3 in opp_dmmode.s
• op_dmode
• dm_read
• setterm <-- most recent/last call

After continuing, returns without any delay to YDB> prompt

***************************************************************************

***************************************************************************

Interleaving the findings from inside gdb, and from outside via strace. 

***************************************************************************

***************************************************************************

NOTE:  On the tcsetattr calls, I am going to DELETE log of the c_oflags, c_cflags, c_line, c_cc -- as they never change

--------------------

------ test 1 ------

--------------------

HANG 10 READ *X HANG 10

After entering command, I immediately get break, with stack as follows:

GDB STACK 

• ?
  
• stkok3
• op_dmmode
• dm_read
• resetterm  <-- this creates strace output below
  

STRACE

ioctl(0, SNDCTL_TMR_START or TCSETS, {

  c_iflags=0x500, binary 0101 0000 0000  (IXON | ICRNL)

  c_lflags=0x8a3b, binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734272633, ...) = -1 EINTR (Interrupted system call)

-- 1st HANG --

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734272633...) = 0

-- hang done --

After continuing, and waiting through HANG, next breakpoint shows stack as follows:

GDB STACK 

• ?
• op_rdone
• iott_rdone
• setterm <-- this generated strace output below.
  

STRACE

ioctl(0, SNDCTL_TMR_START or TCSETS, {

  c_iflags=0x400, binary 0100 0000 0000  <-- IXON (Enable start/stop output control)

  c_lflags=0x8a31, binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)

  c_cc[VMIN]=1, 

  c_cc[VTIME]=8, 

read(0, "a", 1)                         = 1   <--- I pressed 'a' here

After continuing, process waits for me to enter a key.  I enter a letter, and process then reaches breakpoint, showing stack as follows:

GDB STACK

• ?
• op_rdone
• iott_rdone
• resetterm <-- this generated strace output below.

STRACE

ioctl(0, SNDCTL_TMR_START or TCSETS, {

  c_iflags=0x500, binary 0101 0000 0000  <-- IXON  | ICRNL

  c_lflags=0x8a3b, binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)

===2nd HANG =========

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734272643...) = 0

-- hang done --

After continuing, and waiting through hang, next breakpoint shows stack:

GDB STACK

• ?
• stkok3 (in opp_dmode.s)
• op_dmode
• dm_read
• setterm <-- this generated strace output below.

STRACE:

ioctl(0, SNDCTL_TMR_START or TCSETS, {

  c_iflags=0x400, binary 0100 0000 0000  <-- IXON (Enable start/stop output control)

  c_lflags=0x8a31, binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)

  c_cc[VMIN]=1, 

  c_cc[VTIME]=8, 

after continuing, I am returned to YDB> prompt

Summary of test 1:

HANG 10 READ *X HANG 10

• resetterm
  
• hang
• setterm
• read(0, "a", 1)= 1   <--- I pressed 'a' here
• resetterm <-- this generated strace output below.
• hang
• setterm <-- this generated strace output below.
• Returned to YDB> prompt

--------------------

------ test 2 ------

--------------------

HANG 10 READ *X:0.1 HANG 10

First beakpoint encounter shows stack as below.  This is BEFORE hang:

• ?
• stkok3 in opp_dmmode.s
• op_dmode
• dm_read
• setterm <-- most recent/last call

ioctl(0, SNDCTL_TMR_START or TCSETS, {

  c_iflags=0x500, binary 0101 0000 0000  <-- (IXON  | ICRNL)

  c_lflags=0x8a3b, binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)

clock_nanosleep(CLOCK_REALTIME, ... })= -1 EINTR (Interrupted system call)

-- 1st HANG --

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=173427191... ) = 0

-- hang done --

After continuing, and waiting through HANG, next breakpoint reached, showing stack:

GDB STACK:

• ?
• op_rdone
• iott_rdone
• setterm  <-- most recent/last call

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x400, binary 0100 0000 0000  <-- IXON (Enable start/stop output control)

c_lflags=0x8a31, binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)

c_cc[VMIN]=1,   <--- this is not set after overwritten below

c_cc[VTIME]=8, <--- this is not set after overwritten below

After continuing, next breakpoint reached WITHOUT any delay (HANG), showing stack:

• ?
• op_rdone
• iott_rdone
• resetterm <-- most recent/last call

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x500, <-- binary 0101 0000 0000  <-- IXON  | ICRNL

c_lflags=0x8a3b, <-- binary 1000 1010 0011 1011  (ISIG=ON, ICANON=ON, XCASE=OFF, ECHO=ON)

-- 2nd HANG, after READ *X:0.1  --

clock_nanosleep(CLOCK_REALTIME, TIMER_ABSTIME, {tv_sec=1734271927...) = 0

-- hang done --

After continuing, and wanting through HANG, next breakpoint reached, showing stack:

• ?
• stkok3 in opp_dmmode.s
• op_dmode
• dm_read
• setterm <-- most recent/last call

-- resetting IO in preparation for the YDB> prompt --

ioctl(0, SNDCTL_TMR_START or TCSETS, {

c_iflags=0x400, <-- binary 0100 0000 0000  <-- IXON (Enable start/stop output control)

c_lflags=0x8a31, <-- binary 1000 1010 0011 0001 (ISIG=ON, ICANON=OFF, XCASE=OFF, ECHO=OFF)

c_cc[VMIN]=1, 

c_cc[VTIME]=8, 

After continuing, returns without any delay to YDB> prompt

Summary of test 2:

HANG 10 READ *X:0.1 HANG 10

• setterm
• HANG
• setterm
• resetterm
  
• HANG
• setterm <-- most recent/last call
• After continuing, returns without any delay to YDB> prompt

********************************************

Comparison of summaries 1&2

********************************************

Summary of test 1:

HANG 10 READ *X HANG 10

• resetterm
  
• hang
• setterm
• read(0, "a", 1)= 1   <--- I pressed 'a' here
• resetterm <-- this generated strace output below.
• hang
• setterm <-- this generated strace output below.
• Returned to YDB> prompt

Summary of test 2:

HANG 10 READ *X:0.1 HANG 10

• setterm
• HANG
• setterm
• <--- is immediate read occurring here??
  
• resetterm 
• <--- or here??
  
• HANG
• setterm <-- most recent/last call
• After continuing, returns without any delay to YDB> prompt

=================================================

Conclusions:

• It is resetterm() that is putting tty into ICANON=ON mode, which tells the tty system to interpret BACKSPACE as an editing key, and to delete element from input buffor.
  
• Both READ *X and READ *X:0.1 both seem to follow same pattern of setterm, followed by resetterm.
  
• I next need to determine if resetterm is called before or after attempts to read from input buffer

That's all for now. 

[Kevin Toppenberg]

Typo correction:

OK, today I am going to investigate more why setterm() is being called twice.  1st time is appropriate setup, 2nd time overwrites the 1st, and sets CANONICAL mode which consumes backspace.

Should be

OK, today I am going to investigate more why tcsetatter() is being called multiple times.  1st time is appropriate setup, 2nd time overwrites the 1st, and sets CANONICAL mode which consumes backspace.

[Kevin Toppenberg]

Another typo:

In gdb, I set only one breakpoint: setterm.c:setterm

should be:

In gdb, I set breakpoints in all places that call Tcsetattr().  But the only breakpoints actually encountered were setterm and resetterm

KT

[Kevin Toppenberg]

After thinking about this some more, and doing some more stepping through code, I have realized that both:

  HANG 5 READ *X HANG 5      and

  HANG 5 READ *X:0.1 HANG 5

... that both go through iott_rdone.   At the beginning of the file is the setterm(), which turns OFF canonical mode, and at the end of the file is resetterm() which turns it back on.   And in the middle is the reading of the io buffer.

So my suspicion that resetterm() was being called before the IO read is NOT correct.

So there is something different about the path taken through iott_rdone between the two cases.  I need better debugging tools to figure it out.  I can't do it with gdb because it messes with the IO buffer to get user debugging commands.

I think I will have to abandon the docker container and get yottadb compiling in my virtual machine so that I can modify the code to give me hints of what is happening.  I'm thinking about putting assignments into new debug variables at the various decision points (if tests), that I can then query after all the IO stuff is done. 

This is going to take awhile....

KT

[Kevin Toppenberg]

OK, I have gotten yottadb compiling, following the instructions here:

https://gitlab.com/YottaDB/DB/YDB

I then installed vscode, and with the help of chatgpt...

make tasks.json and put in .vscode folder off of <project root>

{
  "version": "2.0.0",
  "tasks": [
    {
      "label": "cmake",
      "type": "shell",
      "command": "cmake",
      "args": [
        "-D",
        "CMAKE_BUILD_TYPE=Debug",
        ".."
      ],
      "options": {
        "cwd": "${workspaceFolder}/build"
      },
      "problemMatcher": []
    },
    {
      "label": "build",
      "type": "shell",
      "command": "make",
      "args": [
        "-j",
        "${env:NUMBER_OF_PROCESSORS}" // or "$(getconf _NPROCESSORS_ONLN)" if you prefer
      ],
      "options": {
        "cwd": "${workspaceFolder}/build"
      },
      "group": {
        "kind": "build",
        "isDefault": true
      },
      "dependsOn": "cmake",
      "problemMatcher": ["$gcc"]
    },
    {
      "label": "install",
      "type": "shell",
      "command": "make",
      "args": [
        "install"
      ],
      "options": {
        "cwd": "${workspaceFolder}/build"
      },
      "problemMatcher": []
    }
  ]
}

then make launch.json, also put into .vscode

{
    "version": "0.2.0",
    "configurations": [
        {
            "name": "Debug",
            "type": "cppdbg",
            "request": "launch",
            "program": "${workspaceFolder}/build/yottadb", // Path to your executable
            "args": ["-dir"], // Command-line arguments
            "stopAtEntry": false,
            "cwd": "${workspaceFolder}", // Working directory for the debugger
            "environment": [],
            "externalConsole": false,
            "MIMode": "gdb",
            "miDebuggerPath": "/usr/bin/gdb", // Or your debugger's path
            "setupCommands": [
                {
                    "description": "Enable pretty-printing for gdb",
                    "text": "-enable-pretty-printing",
                    "ignoreFailures": true
                }
            ],
            "preLaunchTask": "build" // Matches the label of your build task in tasks.json
        }
    ]

}

After than I can go to menu -> run with debugging.  It will open a console and allow me to interact with yottadb, and stop on breakpoints, and inspect variables etc.  Amazing!!

KT

[Kevin Toppenberg]

My original post in this thread was to investigate PASTHRU mode in yottadb, and understand how this relates to the TTY raw mode.

After looking into this for a few weeks, I can say that PASTHRU mode is yottadb method of handling input. It does NOT have anything to do with the TTY subsystem or raw mode.  

I'm just posting this for future reference.

Kevin

[Kevin Toppenberg]

As a final post in this thread, I am going to paste in some debugging notes I made.  I pulled all the data types into one location so I could trace their interconnections.

Kevin

// Fixed-size types, underlying types depend on word size and compiler.

typedef signed char int8_t;

typedef unsigned char uint8_t;

typedef signed short int int16_t;

typedef unsigned short int uint16_t;

typedef signed int int32_t;

typedef int32_t int4; // 4-byte signed integer

typedef unsigned int uint32_t;

typedef uint32_t uint4; // 4-byte unsigned integer

typedef unsigned int wint_t;

typedef int boolean_t; // boolean type

typedef int pid_t;

typedef unsigned long int uint64_t;

typedef uint64_t uint8; // 8-byte unsigned integer

typedef int4 mint;

typedef struct { //io_terminator;

uint4 x;

uint4 mask;

} io_terminator;

typedef __gnuc_va_list va_list;

#define ESC_LEN 16

#define DD_BUFLEN 80

#define bool _Bool

#define GTM_MB_LEN_MAX 4 // maximum bytes we support for multi-byte char

typedef __builtin_va_list __gnuc_va_list;

#define TERM_MSK 0x08002400 /* CR LF ESC */

#define TERM_MSK_UTF8_0 0x08003400 /* add FF */

#define TERM_MSK_UTF8_4 0x00000020 /* NL */

typedef struct { //io_termmask;

uint4 mask[8]; //holds TERM_MSK, TERM_MSK_UTF8_0 etc...

} io_termmask;

typedef struct { //io_pair;

struct io_desc_struct * in;

struct io_desc_struct * out;

} io_pair;

enum tt_which {

ttwhichinvalid,

dmread,

ttread,

ttrdone

};

enum io_dev_state {

dev_never_opened,

dev_closed,

dev_open,

n_io_dev_states

};

enum io_dev_type {

tt, //terminal

mt, //mag tape - obsolete but left so devices aren't renumbered - could reuse

rm, //rms

us, //user device driver

mb, //mail box - obsolete but left so devices aren't renumbered - could reuse

nl, //null device

ff, //fifo device

gtmsocket, //socket device, socket is already used by sys/socket.h

pi, //pipe

n_io_dev_types //terminator

};

// Enumerator codes for supported CHSETs in GT.M.

// This enum table must match the order of the chset_names table defined in mtables.c.

typedef enum {

CHSET_M,

CHSET_UTF8,

CHSET_UTF16,

CHSET_UTF16LE,

CHSET_UTF16BE,

CHSET_ASCII,

CHSET_EBCDIC,

CHSET_BINARY,

CHSET_DEC,

CHSET_HEX,

CHSET_MAX_IDX_ALL // maximum number of CHSETs supported

} gtm_chset_t;

typedef struct io_log_name_struct {

io_desc * iod; // io descriptor <-- io_desc is struct we are in currently

struct io_log_name_struct* next; // linked list

unsigned char len; // name length

char dollar_io[1]; // _$IO hidden variable

} io_log_name;

typedef struct { //mval

unsigned short mvtype;

unsigned char e : 7; //bit field

unsigned char sgn : 1; //bit field

unsigned char fnpc_indx; //Index to fnpc_work area this mval is using

unsigned int utfcgr_indx; //Index to utfcgr_work area this mval is using

int4 m[2];

mstr str;

} mval;

typedef struct dev_dispatch_struct {

//Below are function pointers

//RsltType FnName FnParams

short (*open) (io_log_name*, mval*, int, mval*, uint8);

void (*close) (io_desc*, mval*);

void (*use) (io_desc*, mval*);

int (*read) (mval*, uint8);

int (*rdone) (mint*, uint8);

void (*write) (mstr*);

void (*wtone) (int);

void (*wteol) (int4, io_desc*);

void (*wtff) (void);

void (*wttab) (int4);

void (*flush) (io_desc*);

int (*readfl) (mval*, int4, uint8);

void (*iocontrol) (mstr*, int4, va_list);

void (*dlr_device) (mstr*);

void (*dlr_key) (mstr*);

void (*dlr_zkey) (mstr*);

} dev_dispatch_struct;

typedef int __time_t; //kt added

// POSIX.1b structure for a time value.

// This is like a `struct timeval' but

// has nanoseconds instead of microseconds.

struct timespec {

__time_t tv_sec; // Seconds.

int: 32; // Padding.

long int tv_nsec; // Nanoseconds.

};

typedef struct timespec ABS_TIME;

typedef struct { //tt_interrupt;

enum tt_which who_saved;

unsigned char* buffer_start; // initial stringpool.free

wint_t* buffer_32_start;

int utf8_more;

int dx;

int dx_start;

int dx_instr;

int dx_outlen;

int instr;

int outlen;

int index; // dm_read only

int cl; // dm_read only

int length;

int exp_length;

int recall_index; // index corresponding to input string that was recalled when interrupted

int no_up_or_down_cursor_yet;

int utf8_seen;

boolean_t insert_mode;

ABS_TIME end_time;

unsigned char* zb_ptr;

unsigned char* zb_top;

unsigned short escape_length; // dm_read only

unsigned char escape_sequence[ESC_LEN]; // dm_read only

unsigned char more_buf[GTM_MB_LEN_MAX + 1];

boolean_t timed; //ifdef DEBUG

uint8 nsec_timeout; //ifdef DEBUG

} tt_interrupt;

typedef int mstr_len_t; // Change MSTR_LEN_MAX if this changes

typedef struct { //mstr

unsigned int char_len; // Character length

mstr_len_t len;

char* addr;

} mstr;

typedef struct { //recall_ctxt_t;

int nchars; // In M mode this is # of "unsigned char" sized bytes in the string pointed to by "buff".

// In UTF-8 mode, this is the # of "wint_t" sized codepoints in the string pointed to by "buff".

int width; // Total display width of the "nchars" characters

int bytelen; // Total byte length of the "nchars" characters

char* buff; // In M mode, this points to the array of bytes comprising the input string.

// In UTF-8 mode, this points to an array of "wint_t" sized codepoints comprising the input string.

} recall_ctxt_t;

typedef unsigned char cc_t;

typedef unsigned int speed_t;

typedef unsigned int tcflag_t;

#define NCCS 32

struct termios {

tcflag_t c_iflag; // input mode flags (note: values below are OCTAL)

// IGNBRK 0000001 Ignore break condition.

// BRKINT 0000002 Signal interrupt on break.

// IGNPAR 0000004 Ignore characters with parity errors.

// PARMRK 0000010 Mark parity and framing errors.

// INPCK 0000020 Enable input parity check.

// ISTRIP 0000040 Strip 8th bit off characters.

// INLCR 0000100 Map NL to CR on input.

// IGNCR 0000200 Ignore CR.

// ICRNL 0000400 Map CR to NL on input.

// IUCLC 0001000 Map uppercase characters to lowercase on input (not in POSIX).

// IXON 0002000 Enable start/stop output control.

// IXANY 0004000 Enable any character to restart output.

// IXOFF 0010000 Enable start/stop input control.

// IMAXBEL 0020000 Ring bell when input queue is full (not in POSIX).

// IUTF8 0040000 Input is UTF8 (not in POSIX).

tcflag_t c_oflag; // output mode flags //NOTE: doesn't seem to be used in yottadb

tcflag_t c_cflag; // control mode flags //NOTE: doesn't seem to be used in yottadb

tcflag_t c_lflag; // local mode flags (note: values below are OCTAL)

// ISIG 0001 Enable signals.

// ICANON 0002 Canonical input (erase and kill processing).

// XCASE 0004

// ECHO 0010 Enable echo.

// ECHOE 0020 Echo erase character as error-correcting backspace

// ECHOK 0040 Echo KILL.

// ECHONL 0100 Echo NL.

// NOFLSH 0200 Disable flush after interrupt or quit.

// TOSTOP 0400 Send SIGTTOU for background output.

cc_t c_line; // line discipline //NOTE: doesn't seem to be used in yottadb

cc_t c_cc[NCCS]; // control characters

speed_t c_ispeed; // input speed

speed_t c_ospeed; // output speed

};

//============================================================

//============================================================

//TT (TeleType/terminal) device struct

//============================================================

//============================================================

typedef struct { //d_tt_struct

uint4 in_buf_sz; // size of read buffer

uint4 ext_cap;

io_terminator enbld_outofbands; // enabled out-of-band chars

uint4 term_ctrl; //bit flags: In code, often assigned to var named 'mask'

// TRM_MODIFIERS 0

// TRM_EDITMODE 1

// TRM_NOTYPEAHD 2

// TRM_READSYNC 4

// TRM_PROMPT 8

// TRM_PASTHRU 16

// TRM_ESCTRMOVR 32

// TRM_ESCAPE 64

// TRM_NOECHO 128

// TRM_CONVERT 256

io_termmask mask_term; // array of terminators: uint4 mask[8];

boolean_t default_mask_term; // mask_term is the default. If TRUE, uses default. If FALSE, uses mask_term[]

int fildes;

struct termios* ttio_struct;

tt_interrupt tt_state_save; // in case job interrupt

boolean_t mupintr; // read was interrupted

char* ttybuff; // buffer for tty

volatile char* tbuffp; // next open space in buffer

recall_ctxt_t* recall_array; // if EDITING enabled, this points to MAX_RECALL-sized array of previously input strings.

int recall_index; // Offset into circular "recall_array" pointing one past to most recent input.

volatile boolean_t timer_set; // text flush timer is set

volatile boolean_t write_active; // we are in write -- postpone flush by timer

boolean_t canonical;

boolean_t discard_lf; // UTF8 mode - previous char was CR so ignore following LF

boolean_t done_1st_read; // UTF8 mode - check for BOM if not

pid_t setterm_done_by; // if non-zero, points to pid that did "setterm"; used to later invoke "resetterm" if needed.

} d_tt_struct;

//============================================================

//============================================================

//IO DESCriptor

//============================================================

//============================================================

typedef struct io_desc_struct { //io_desc <--> 'struct io_desc_struct';

mstr error_handler;

unsigned int length;

unsigned int width;

bool perm; // permanent

bool wrap; // if FALSE trunc

boolean_t fflf; // if TRUE "write #" emits FF+LF on disk files, otherwise bare LF

gtm_chset_t ichset;

gtm_chset_t ochset;

int4 write_filter;

unsigned char esc_state;

d_tt_struct * dev_sp; //"Device Struct Ptr"(//kt) //used as 'tt_ptr' //kt added type. was void* before

boolean_t newly_created;

enum io_dev_type type;

enum io_dev_state state;

io_pair pair;

struct io_log_name_struct* trans_name;

struct io_log_name_struct* name;

struct dev_dispatch_struct* disp_ptr;

struct { //dollar

unsigned int x;

unsigned int y;

unsigned short zeof;

unsigned short za;

unsigned char zb[ESC_LEN];

char device[DD_BUFLEN];

char* devicebuffer;

int devicebufferlen;

} dollar;

//

} io_desc; //io_desc <--> 'struct io_desc_struct'

//Globally scoped vars:

io_desc* active_device;

io_pair io_curr_device, io_std_device;