ValueError: filedescriptor out of range in select()

[Simon Brandhorst]

I am trying a medium scale computation. Unfortunately it breaks every time with error messages I do not know how to interpret.

They may or may not be bugs in sage. Here is a traceback




--------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-6-1e340a4a9be4> in <module>()
----> 1 classify_ord_pe(L,Integer(2),Integer(2),'results/order4.txt','a')

/home/simon/.sage/temp/k3/2030/K3_aut_classification.sageWSsnuG.py in classify_ord_pe(L, p, e, file_name, rw)
    258         print(" ")
    259         cofix = cofixed[k].twist(-_sage_const_1 )
--> 260         for Aa in k3_prime_power(fix.genus(), p, e):
    261             A, a, Oa = Aa
    262             actsg = MaximalK3surfaceAut(A, cofix, a, Oa)

/home/simon/.sage/temp/k3/2030/prime_power.sagez0p9M2.py in k3_prime_power(genus, prime, e)
    138             signatures += [[ranks_E[k]]*(weights[k]//_sage_const_2 ) for k in range(_sage_const_1 ,n)]
    139         signatures[-_sage_const_1 ][_sage_const_0 ] -= _sage_const_1
--> 140         for act in prime_power_actions(genus,prime,ranks,signatures):
    141             yield act
    142

/home/simon/.sage/temp/k3/2030/prime_power.sagez0p9M2.py in prime_power_actions(genus, p, ranks, signatures)
    278                     # recurse
    279                     # print(R)
--> 280                     for N in prime_power_actions(R, p, ranks[:-_sage_const_1 ], R_signatures):
    281                         N, fN, GN = N
    282                         ext = extensions(M, fM, N, fN, GM, GN, glue_order, p)

/home/simon/.sage/temp/k3/2030/prime_power.sagez0p9M2.py in prime_power_actions(genus, p, ranks, signatures)
    278                     # recurse
    279                     # print(R)
--> 280                     for N in prime_power_actions(R, p, ranks[:-_sage_const_1 ], R_signatures):
    281                         N, fN, GN = N
    282                         ext = extensions(M, fM, N, fN, GM, GN, glue_order, p)

/home/simon/.sage/temp/k3/2030/prime_power.sagez0p9M2.py in prime_power_actions(genus, p, ranks, signatures)
    196     if len(ranks) == _sage_const_1 :
    197         # there is nothing to glue
--> 198         for M in genus.representatives():
    199             fM = M ** _sage_const_0
    200             M = IntegralLattice(M)

/home/simon/sage/local/lib/python2.7/site-packages/sage/quadratic_forms/genera/genus.pyc in representatives(self, backend, algorithm)
   3161                 if self.signature_pair()[0] == 0:
   3162                     e *= ZZ(-1)
-> 3163                 Q = QuadraticForm(e*self.representative())
   3164                 seeds = [Q]
   3165                 for p in self.spinor_generators(proper=False):

/home/simon/sage/local/lib/python2.7/site-packages/sage/quadratic_forms/genera/genus.pyc in representative(self)
   3010         """
   3011         if self._representative is None:
-> 3012             self._compute_representative()
   3013         return self._representative
   3014

/home/simon/sage/local/lib/python2.7/site-packages/sage/quadratic_forms/genera/genus.pyc in _compute_representative(self, LLL)
   3085                     gp = Gp()
   3086                     m = pari(L)
-> 3087                     gp.read(SAGE_EXTCODE + "/pari/simon/qfsolve.gp")
   3088                     m = gp.eval('qflllgram_indefgoon(%s)'%m)
   3089                     # convert the output string to sage

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/interface.pyc in read(self, filename)
    222             sage: os.unlink(filename)
    223         """
--> 224         self.eval(self._read_in_file_command(filename))
    225
    226     def _read_in_file_command(self, filename):

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/expect.pyc in eval(self, code, strip, synchronize, locals, allow_use_file, split_lines, **kwds)
   1349                 elif split_lines:
   1350                     return '\n'.join([self._eval_line(L, allow_use_file=allow_use_file, **kwds)
-> 1351                                         for L in code.split('\n') if L != ''])
   1352                 else:
   1353                     return self._eval_line(code, allow_use_file=allow_use_file, **kwds)

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/gp.pyc in _eval_line(self, line, allow_use_file, wait_for_prompt, restart_if_needed)
    443         a = Expect._eval_line(self, line,
    444                               allow_use_file=allow_use_file,
--> 445                               wait_for_prompt=wait_for_prompt)
    446         if a.find("the PARI stack overflows") != -1:
    447             verbose("automatically doubling the PARI stack and re-executing current input line")

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/expect.pyc in _eval_line(self, line, allow_use_file, wait_for_prompt, restart_if_needed)
    914         try:
    915             if self._expect is None:
--> 916                 self._start()
    917             E = self._expect
    918             try:

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/gp.pyc in _start(self, alt_message, block_during_init)
    241
    242     def _start(self, alt_message=None, block_during_init=True):
--> 243         Expect._start(self, alt_message, block_during_init)
    244         # disable memory debugging: those warnings can only confuse our
    245         # interface

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/expect.pyc in _start(self, alt_message, block_during_init)
    517
    518         try:
--> 519             self._expect.expect(self._prompt)
    520         except (pexpect.TIMEOUT, pexpect.EOF) as msg:
    521             self._expect = None

/home/simon/sage/local/lib/python2.7/site-packages/pexpect/spawnbase.pyc in expect(self, pattern, timeout, searchwindowsize, async_, **kw)
    339         compiled_pattern_list = self.compile_pattern_list(pattern)
    340         return self.expect_list(compiled_pattern_list,
--> 341                 timeout, searchwindowsize, async_)
    342
    343     def expect_list(self, pattern_list, timeout=-1, searchwindowsize=-1,

/home/simon/sage/local/lib/python2.7/site-packages/pexpect/spawnbase.pyc in expect_list(self, pattern_list, timeout, searchwindowsize, async_, **kw)
    367             return expect_async(exp, timeout)
    368         else:
--> 369             return exp.expect_loop(timeout)
    370
    371     def expect_exact(self, pattern_list, timeout=-1, searchwindowsize=-1,

/home/simon/sage/local/lib/python2.7/site-packages/pexpect/expect.pyc in expect_loop(self, timeout)
    109                     return self.timeout()
    110                 # Still have time left, so read more data
--> 111                 incoming = spawn.read_nonblocking(spawn.maxread, timeout)
    112                 if self.spawn.delayafterread is not None:
    113                     time.sleep(self.spawn.delayafterread)

/home/simon/sage/local/lib/python2.7/site-packages/pexpect/pty_spawn.pyc in read_nonblocking(self, size, timeout)
    455         # * https://github.com/pexpect/pexpect/pull/304
    456         # * http://trac.sagemath.org/ticket/10295
--> 457         if select(0):
    458             try:
    459                 incoming = super(spawn, self).read_nonblocking(size)

/home/simon/sage/local/lib/python2.7/site-packages/pexpect/pty_spawn.pyc in select(timeout)
    447         else:
    448             def select(timeout):
--> 449                 return select_ignore_interrupts([self.child_fd], [], [], timeout)[0]
    450
    451         # If there is data available to read right now, read as much as

/home/simon/sage/local/lib/python2.7/site-packages/pexpect/utils.pyc in select_ignore_interrupts(iwtd, owtd, ewtd, timeout)
    141     while True:
    142         try:
--> 143             return select.select(iwtd, owtd, ewtd, timeout)
    144         except InterruptedError:
    145             err = sys.exc_info()[1]

ValueError: filedescriptor out of range in select()

[Dima Pasechnik]

Could it be a failed attempt to recover from a PARI/GP stackoverflow?
Just guessing.

> --
> You received this message because you are subscribed to the Google Groups "sage-devel" group.
> To unsubscribe from this group and stop receiving emails from it, send an email to sage-devel+...@googlegroups.com.
> To post to this group, send email to sage-...@googlegroups.com.
> Visit this group at https://groups.google.com/group/sage-devel.
> For more options, visit https://groups.google.com/d/optout.

[Simon Brandhorst]

Trying to provoke more errors:

sage: from sage.env import SAGE_EXTCODE
sage: for k in range(10000):
....:     gp = Gp()
....:     gp.read(SAGE_EXTCODE + "/pari/simon/qfsolve.gp")
....:     k

breaks at k = 1922 with

---------------------------------------------------------------------
OSError                             Traceback (most recent call last)
<ipython-input-5-38e8adf2a837> in <module>()
      1 for k in range(Integer(10000)):
      2     gp = Gp()
----> 3     gp.read(SAGE_EXTCODE + "/pari/simon/qfsolve.gp")
      4     k
      5

    501                         # Work around https://bugs.python.org/issue1652
    502                         preexec_fn=lambda: signal.signal(signal.SIGPIPE, signal.SIG_DFL),
--> 503                         quit_string=self._quit_string())
    504             except (ExceptionPexpect, pexpect.EOF) as e:
    505                 # Change pexpect errors to RuntimeError

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/sagespawn.pyx in sage.interfaces.sagespawn.SageSpawn.__init__ (build/cythonized/sage/interfaces/sagespawn.c:1948)()
     63         kwds.setdefault("maxread", 4194304)
     64
---> 65         with ContainChildren(silent=True):
     66             spawn.__init__(self, *args, **kwds)
     67

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/sagespawn.pyx in sage.interfaces.sagespawn.SageSpawn.__init__ (build/cythonized/sage/interfaces/sagespawn.c:1902)()
     64
     65         with ContainChildren(silent=True):
---> 66             spawn.__init__(self, *args, **kwds)
     67
     68         self.delaybeforesend = None

/home/simon/sage/local/lib/python2.7/site-packages/pexpect/pty_spawn.pyc in __init__(self, command, args, timeout, maxread, searchwindowsize, logfile, cwd, env, ignore_sighup, echo, preexec_fn, encoding, codec_errors, dimensions, use_poll)
    202             self.name = '<pexpect factory incomplete>'
    203         else:
--> 204             self._spawn(command, args, preexec_fn, dimensions)
    205         self.use_poll = use_poll
    206

/home/simon/sage/local/lib/python2.7/site-packages/pexpect/pty_spawn.pyc in _spawn(self, command, args, preexec_fn, dimensions)
    301
    302         self.ptyproc = self._spawnpty(self.args, env=self.env,
--> 303                                      cwd=self.cwd, **kwargs)
    304
    305         self.pid = self.ptyproc.pid

/home/simon/sage/local/lib/python2.7/site-packages/sage/interfaces/sagespawn.pyx in sage.interfaces.sagespawn.SageSpawn._spawnpty (build/cythonized/sage/interfaces/sagespawn.c:2133)()
     80             SagePtyProcess.spawn(...)
     81         """
---> 82         ptyproc = SagePtyProcess.spawn(args, **kwds)
     83         ptyproc.quit_string = self.quit_string
     84         return ptyproc

/home/simon/sage/local/lib/python2.7/site-packages/ptyprocess/ptyprocess.pyc in spawn(cls, argv, cwd, env, echo, preexec_fn, dimensions)
    218
    219         if use_native_pty_fork:
--> 220             pid, fd = pty.fork()
    221         else:
    222             # Use internal fork_pty, for Solaris

/home/simon/sage/local/lib/python2.7/pty.pyc in fork()
    106
    107     master_fd, slave_fd = openpty()
--> 108     pid = os.fork()
    109     if pid == CHILD:
    110         # Establish a new session.

OSError: [Errno 11] Resource temporarily unavailable
sage:                                                               
**********************************************************************

Oops, Sage crashed. We do our best to make it stable, but...
........

[Simon Brandhorst]

I can consistently reproduce that error. Tried again and crashed at k = 1888

and

k = 1871

and

k = 1842

maybe I am hitting some preset limit?

[Vincent Delecroix]

Could you open a trac ticket? This is definitely a bug
(most probably in `interfaces/expect.py`).

[Jeroen Demeyer]

On 2019-03-03 11:43, Vincent Delecroix wrote:
> Could you open a trac ticket? This is definitely a bug
> (most probably in `interfaces/expect.py`).

I wouldn't call it a bug. You're opening 10000 independent GP sessions,
which each need a few file descriptors. It's normal that you run out of
file descriptors after a while.

[Vincent Delecroix]

Is this supposed to crash Sage?

[Vincent Delecroix]

Can you try *directly* at k=1842 (no loop). As Jeroen pointed
out, each gp session opens a file descriptor.

[Jeroen Demeyer]

On 2019-03-03 11:48, Vincent Delecroix wrote:
> Is this supposed to crash Sage?

OK, I missed that part. The fact that you run out of file descriptors is
not a bug, but the fact that it crashes Sage is a bug.

[Simon Brandhorst]

Shouldn't the gp session be closed afterwards? After all it is not used/referenced anymore?

So that the file descriptor gets freed again?

If not what is the recommended way to start computations in gp (or gap/magma ....)?

Say in some method in some class? give it some global gp instance as a parameter? Is there such a thing?

[Jeroen Demeyer]

On 2019-03-03 12:46, Simon Brandhorst wrote:
> Shouldn't the gp session be closed afterwards? After all it is not
> used/referenced anymore?

But maybe it's not closed *immediately*. It would be interesting to do
the same experiment with a time.sleep(0.1) or a gc.collect() in the loop.

[Simon Brandhorst]

Ah I see and to answer my own question using grep:
from sage.interfaces.gp import gp
is probably what I should do.

[Jeroen Demeyer]

On 2019-03-03 12:46, Simon Brandhorst wrote:

> Shouldn't the gp session be closed afterwards? After all it is not
> used/referenced anymore?
> So that the file descriptor gets freed again?
> If not what is the recommended way to start computations in gp

The recommended way is not to use the GP pexpect interface at all but
use the PARI library interface accessed by the "pari" object.

What is it that you're *actually* trying to do?

[Simon Brandhorst]

On Sunday, March 3, 2019 at 12:50:42 PM UTC+1, Jeroen Demeyer wrote:

But maybe it's not closed *immediately*. It would be interesting to do
the same experiment with a time.sleep(0.1) or a gc.collect() in the loop.

I tried that

sage: for k in range(10000):

....:     gp = Gp()
....:     gp.read(SAGE_EXTCODE + "/pari/simon/qfsolve.gp")
....:     k

....:     if k % 1000 == 0:
....:         gc.collect()

And indeed this time it worked for the first 3266 loops (instead of just 1800) and then sage crashes with

--------------------------------------------------------------------

OSError                             Traceback (most recent call last)

<ipython-input-4-2a823e1e337d> in <module>()

      1 for k in range(Integer(10000)):
      2     gp = Gp()
----> 3     gp.read(SAGE_EXTCODE + "/pari/simon/qfsolve.gp")
      4

    k
      5     if k % Integer(1000) == Integer(0):

^[[35;1R

**********************************************************************

Oops, Sage crashed. We do our best to make it stable, but...

A crash report was automatically generated wdth the following information:
  - A verbatim copy of the crash traceback.
  - A copy of your input history during this session.
  - Data on your current Sage configuration.

It was left in the file named:
    '/home/simon/.sage/ipython-5.0.0/Sage_crash_report.txt'
If you can email this file to the developers, the information in it will help
them in understanding and correcting the problem.

You can mail it to: sage-support at sage-support@googlegroups.com
with the subject 'Sage Crash Report'.

If you want to do it now, the following command will work (under Unix):
mail -s 'Sage Crash Report' sage-support@googlegroups.com < /home/simon/.sage/ipython-5.0.0/Sage_crash_report.txt

In your email, please also include information about:
- The operating system under which the crash happened: Linux, macOS, Windows,
  other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2,
  Windows 10 Pro), and whether it is 32-bit or 64-bit;
- How Sage was installed: using pip or conda, from GitHub, as part of
  a Docker container, or other, providing more detail if possible;
- How to reproduce the crash: what exact sequence of instructions can one
  input to get the same crash? Ideally, find a minimal yet complete sequence
  of instructions that yields the crash.

To ensure accurate tracking of this issue, please file a report about it at:
http://trac.sagemath.org

Hit <Enter> to quit (your terminal may close):

[Simon Brandhorst]

On Sunday, March 3, 2019 at 1:09:45 PM UTC+1, Jeroen Demeyer wrote:

What is it that you're *actually* trying to do?

Use the gp code by Denis Simon in  

/src/ext/pari/simon

didnt know how to feed that to pari, so I used gp. And that is what I saw in

schemes/elliptic_curves/gp_simon.py:        gp.read("resultant3.gp")

as well.

[E. Madison Bray]

I'm not sure I'd even call that a bug, so long as it crashes "cleanly"
(provides a relevant traceback and a non-zero exit code). Running out
of file descriptors is a pretty extreme condition and possible that
even just in the course of trying to do something like write to a log
file or a dup a file descriptor for some reason (there is all kinds of
I/O magic that goes into making a complex REPL like IPython work, for
example) you could then bring down the interpreter.

At most I'd say maybe the Expect interfaces could handle this type of
error better.