Asynchronous Non blocking socket programming in C

[subrama...@gmail.com]

Hi,

I'm working in a POSIX environment and I'm trying to run an example of an asynchronous non-blocking socket program using C. I could not find complete examples in my futile attempt to search online hence I put together all the pieces myself and attempted a simple synchronous socket server - client model which works well.

So now I went onto create a server - client asynchronous socket model but I am unable to make it work as the fcntl call always returns with an error that the file descriptor is invalid. I've provided my server & client programs below for understanding.

server.c

#DEFINE _XOPEN_SOURCE_EXTENDED 1 /* Needed for POSIX environment to pick up various defines */
#INCLUDE<UNISTD.H>
#INCLUDE<FCNTL.H>
#INCLUDE<TAL.H>
#INCLUDE<STDIO.H>
#INCLUDE<STDLIB.H>
#INCLUDE<STRING.H>
#INCLUDE<ERRNO.H>
#INCLUDE<TIME.H>
#INCLUDE<SYS/TIME.H>
#INCLUDE<SYS/SOCKET.H>
#INCLUDE<ARPA/INET.H>
#INCLUDE<NETINET/IN.H>
#INCLUDE<NETINET/IP.H>
#INCLUDE<NETINET/TCP.H>
#INCLUDE<NETDB.H>
#INCLUDE<SIGNAL.H>
#INCLUDE<TIME.H>
void DieWithError(char *errorMessage);
void recvstuff();
void printstuff();

void delay(int number_of_seconds)
{
int milli_seconds = 1000 * number_of_seconds;
clock_t start_time = clock();
while (clock() < start_time + milli_seconds)
;
}

int fd, newfd, bytesread, bufflen;
struct sockaddr_in addr, claddr;

#DEFINE BUFF_LEN 1024

change msg_buff[BUFF_LEN+1];
signed int clln;

int main(int argc, char **argv)
{
struct sigaction handler;
handler.sa_handler = printstuffl
if(sigfillser(&handler.sa_mask) < 0)
DieWithError("sigfillset() failed\n");

handler.sa_flags=0;

if(sigaction(SIGIO, &handler, 0) < 0)
DieWithError("sigaction() failed\n");

printf("Server starting \n");

if((fd=socket(AF_INET,SOCK_STREAM,0)) < 0)
{
printf("socket() call failed [errno %d]\n", errno);
exit(1);
}

if (fcntl(fd, F_SETOWN, getpid() < 0)
printf("SETOWN failed [errno %d]\n", errno);

if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0)
printf("NONBLOCK failed [errno %d]\n",errno);

if(argc >1)
addr.sin_port = (short)atoi(argv[1]);
else
addr.sin_port=htons(7779);
addr.sin_family=AF_INET;
addr.sin_addr.s_addr=htonl(INADDR_ANY);

claddr.sin_port=htons(7777);
claddr.sin_family=AF_INET;
claddr.sin_addr.s_addr=htonl(gethostid());
clln = sizeof(claddr);

if(bind(fd,(struct sockaddr *)&addr, sizeof(addr)) < 0)
{
printf("bind() call failed [errno %d] \n", errno);
exit(1);
}

if(listen(fd,5) < 0)
{
printf("listen() call failed [errno %d] \n", errno);
exit(1);
}

if((newfd=accept(fd, (struct sockaddr *)&claddr, &clln)) < 0)
{
printf ("accept() call failed [errno %d] \n", errno);
exit(1);
}
else
{
recvstuff();
}
}

void recvstuff()
{
if ((bytesread = recv(newfd, msg_buff, sizeof(msg_buff),0)) <= 0)
{
if (bytesread < 0)
{
printf("recv() call failed [errno %d]\n", errno);
exit(1);
}
else
{
printf(" Zero bytes received \n");
exit(1);
}
}
else
{
delay(10);
printstuff();
}
}

void printstuff()
{
msg_buff[bytesread] = '\0';
printf("Message received = %s", msg_buff);
recvstuff();
}

void DieWithError(char *errorMessage)
{
perror(errorMessage);
exit(1);
}





client.c

#DEFINE _XPOEN_SOURCE_EXTENDED 1
#INCLUDE<UNISTD.H>
#INCLUDE<FCNTL.H>
#INCLUDE<TAL.H>
#INCLUDE<STDIO.H>
#INCLUDE<STDLIB.H>
#INCLUDE<STRING.H>
#INCLUDE<ERRNO.H>
#INCLUDE<SYS/SOCKET.H>
#INCLUDE<ARPA/INET.H>
#INCLUDE<NETINET/IN.H>
#INCLUDE<NETINET/IP.H>
#INCLUDE<NETINET/TCP.H>
#INCLUDE<NETDB.H>

void DieWithError(char *errorMessage);
int fd, bytesread;
pid_t pid;
struct sockaddr_in addr;
#DEFINE BUFF_LEN 1024
char msg_buff[BUFF_LEN+1];

int main(int argc, char **argv)
{
printf("Requester starting \n");
if ((fd=socket(AF_INET, SOCK_STREAM,0))<0)
{
printf("socket() call failed [errno %d]\n", errno);
exit(1);
}

if(argc >1)
addr.sin_port = (short)atoi(argv[1]);
else
addr.sin_port=htons(7777);
addr.sin_family=AF_INET;
addr.sin_addr.s_addr=htonl(gethostid());

sraddr.sin_port=htons(7779);
sraddr.sin_family=AF_INET;
sraddr.sin_addr.s_addr=htonl(gethostid());

if(connect(fd,(struct sockaddr *)&sraddr, sizeof(sraddr)) < 0)
{
printf("connect() failed [errno %d]\n", errno);
exit(1);
}

pid = getpid();

if (fcntl(fd, F_SETOWN, getpid() < 0)
printf("SETOWN failed [errno %d]\n", errno);

if (fcntl(fd, F_SETFL, O_NONBLOCK) < 0)
printf("NONBLOCK failed [errno %d]\n",errno);

while(1)
{
printf("?");
fgets(msg_buff, BUFF_LEN, stdin);
if(feof(stdin))
exit(0);
bytesread = (int)strlen(msg_buff);

if(send(fd,msg_buff, sizeof(msg_buff), 0) < 0)
{
printf("send() call failed");
exit(1);
}

if((bytesread = recv(fd,msg_buff,sizeof(msg_buff),0)) <=0)
{
if(bytesread==0)
printf("Server closed communication\n");
else
printf("recv() call failed\n");
exit(1);
}
printf("Read %d bytes back from server \n", bytesread);
}
}

void DieWithError(char *errorMessage)
{
perror(errorMessage);
exit(1);
}


NOTE: The above two programs have been modified from the synchronous socket program that I had created earlier. The asynchronous program has not worked yet.

I always get an error in step FCNTL SETOWN - errno 4009 which is EBADF -> Not a valid file descriptor. This is the same error in both server and requester. I'm not sure if FCNTL is allowed on sockets? Is it only allowed on readable files? Please confirm.

DISCLAIMER: I work in a restrictive environment and hence I was not able to copy the code outside, this is a best effort reproduction of the exact same code but as I had typed out the one above - it might have slight typos - please pardon me for the same. I hope the information is sufficient to provide your comments, do let me know if you need more info.

Thanks,
Shiva.

[Kaz Kylheku]

On 2018-03-25, subrama...@gmail.com <subrama...@gmail.com> wrote:
> Hi,
>
> I'm working in a POSIX environment and I'm trying to run an example of an asynchronous non-blocking socket program using C. I could not find complete examples in my futile attempt to search online hence I put together all the pieces myself and attempted a simple synchronous socket server - client model which works well.
>
> So now I went onto create a server - client asynchronous socket model but I am unable to make it work as the fcntl call always returns with an error that the file descriptor is invalid. I've provided my server & client programs below for understanding.
>
> server.c
>
> #DEFINE _XOPEN_SOURCE_EXTENDED 1 /* Needed for POSIX environment to pick up various defines */
> #INCLUDE<UNISTD.H>

The C language is case-sensitive; there is no #DEFINE.

This cruft shouldn't compile.

$ gcc -x c -
#DEFINE FOO BAR
^D
<stdin>:1:2: error: invalid preprocessing directive #DEFINE

[Ben Bacarisse]

subrama...@gmail.com writes:
<snip>

> NOTE: The above two programs have been modified from the synchronous
> socket program that I had created earlier. The asynchronous program
> has not worked yet.

The code can't be compiled. You'll get much more help if you present a
example that can be. I did what I so that server.c went through, but
then it fails with a different error to the one you report, so I might
end up investigating something quite different.

> I always get an error in step FCNTL SETOWN - errno 4009 which is EBADF
> -> Not a valid file descriptor. This is the same error in both server
> and requester. I'm not sure if FCNTL is allowed on sockets? Is it only
> allowed on readable files? Please confirm.

I don't get that error. I get:

Server starting
accept() call failed [errno 11]

but that's from an edited version of your program. (BTW, print human
readable error strings rather than simply printing errno.)

--
Ben.

[Rainer Weikusat]

subrama...@gmail.com writes:

['pseudo-code']

> I always get an error in step FCNTL SETOWN - errno 4009 which is EBADF
> -> Not a valid file descriptor. This is the same error in both server
> and requester. I'm not sure if FCNTL is allowed on sockets? Is it only
> allowed on readable files? Please confirm.

JFTR: EBADF is 9, not 4009.

fcntl, including F_SETOWN, works on sockets.

[Jorgen Grahn]

On Sun, 2018-03-25, subrama...@gmail.com wrote:
...
> struct sockaddr_in addr, claddr;
...
> if((fd=socket(AF_INET,SOCK_STREAM,0)) < 0)
...

> if(argc >1)
> addr.sin_port = (short)atoi(argv[1]);
> else
> addr.sin_port=htons(7779);
> addr.sin_family=AF_INET;
> addr.sin_addr.s_addr=htonl(INADDR_ANY);

You still see this kind of code, but it has been obsolete for fifteen
years! Use getaddrinfo(3) instead, so you're not limited to IPv4; so
the user can specify hostnames instead of addresses; plus other
benefits.

/Jorgen

--
// Jorgen Grahn <grahn@ Oo o. . .
\X/ snipabacken.se> O o .

[Jorgen Grahn]

On Sun, 2018-03-25, subrama...@gmail.com wrote:

> Hi,
>
> I'm working in a POSIX environment and I'm trying to run an example
> of an asynchronous non-blocking socket program using C. I could not
> find complete examples in my futile attempt to search online

I haven't found one either (although I haven't looked very closely).
The nginx web proxy might be one.

The short version[0] of non-blocking TCP sockets: you sit waiting on
select(), poll(), or Linux epoll() or something, and:

a. The socket becomes readable:
- Read and save 0--n bytes of data.
- If you now have 1 or more complete application-level,
messages, validate them and enqueue them for processing.
- If you have a partial message, save it until next time.

b. The socket becomes writable:
- If you have nothing to say, don't listen for writability.
- If you have a backlog of data to send, try to send some of it.
- If you have an application-level message to send, try to
send it. If you cannot send it all, add the rest to the
backlog.

c. Nothing happens on the socket for a longish time.
- Consider closing it as an application-level timeout.

In other words, it's a state machine which consumes a stream of bytes
and emits a stream of bytes. Then you have to fit that somehow to
the application-level logic and protocol.

/Jorgen

[0] I'm ignoring things like handling accept(), connect() and name
resolution without blocking.

[Rainer Weikusat]

Jorgen Grahn <grahn...@snipabacken.se> writes:
> On Sun, 2018-03-25, subrama...@gmail.com wrote:
>> Hi,
>>
>> I'm working in a POSIX environment and I'm trying to run an example
>> of an asynchronous non-blocking socket program using C. I could not
>> find complete examples in my futile attempt to search online
>
> I haven't found one either (although I haven't looked very closely).
> The nginx web proxy might be one.

Not a particularly sensible example, but here's an O_ASYNC UDP server
(I strongly suspect there are some bugs in it):

-------
#include <errno.h>
#include <fcntl.h>
#include <netinet/in.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <unistd.h>

enum {
PORT = 65261
};

static int sk;

static int die(char const *sysc)
{
perror(sysc);
exit(1);
}

static int open_socket(unsigned port)
{
struct sockaddr_in sin;
int sk, rc;

sk = socket(AF_INET, SOCK_DGRAM, 0);
sk != -1 || die("socket");

sin.sin_family = AF_INET;
sin.sin_addr.s_addr = INADDR_ANY;
sin.sin_port = htons(port);
rc = bind(sk, (struct sockaddr *)&sin, sizeof(sin));
rc != -1 || die("bind");

return sk;
}

static void block_sigio(void)
{
sigset_t ss;

sigemptyset(&ss);
sigaddset(&ss, SIGIO);
sigprocmask(SIG_BLOCK, &ss, NULL);
}

static void setup_async(int sk)
{
fcntl(sk, F_SETOWN, getpid());
fcntl(sk, F_SETFL, fcntl(sk, F_GETFL) | O_ASYNC | O_NONBLOCK);
}

static void read_msgs(int unused)
{
char msg[65536];
ssize_t nr;

while (nr = read(sk, msg, sizeof(msg)), nr != -1) {
if (nr) {
if (msg[nr - 1] != '\n') msg[nr++] = '\n';
printf("recvd: %.*s", (int)nr, msg);
} else
puts("recvd 0-length message");
}

errno == EAGAIN || die("read");
}

static void setup_sgio_handler(void)
{
struct sigaction sa;

sigemptyset(&sa.sa_mask);
sa.sa_flags = 0;
sa.sa_handler = read_msgs;
sigaction(SIGIO, &sa, NULL);
}

static void unblock_sigio(void)
{
sigset_t ss;

sigemptyset(&ss);
sigaddset(&ss, SIGIO);
sigprocmask(SIG_UNBLOCK, &ss, NULL);
}

int main(void)
{
sk = open_socket(PORT);

block_sigio();
setup_async(sk);
setup_sgio_handler();
unblock_sigio();

while (1) pause();

return 0;
}

[James K. Lowden]

On Sun, 25 Mar 2018 08:23:17 -0700 (PDT)
subrama...@gmail.com wrote:

> I'm working in a POSIX environment and I'm trying to run an example
> of an asynchronous non-blocking socket program using C. I could not
> find complete examples in my futile attempt to search online hence I
> put together all the pieces myself and attempted a simple synchronous
> socket server - client model which works well.

At www.freetds.org you'll find an asynchronous client. Not a minimal
example, but the network functions are all restricted to one file.

From experience I can tell you that network programming cannot be
learned from online sources. You need books. I recommend

The Linux Programming Interface by Michael Kerrisk
UNIX Network Programming by W. Richard Stevens

If you can, read all Stevens' books. Worst case, you'll learn more
than you need to know.

--jkl

[Lew Pitcher]

James K. Lowden wrote:

> On Sun, 25 Mar 2018 08:23:17 -0700 (PDT)
> subrama...@gmail.com wrote:
>
>> I'm working in a POSIX environment and I'm trying to run an example
>> of an asynchronous non-blocking socket program using C. I could not
>> find complete examples in my futile attempt to search online hence I
>> put together all the pieces myself and attempted a simple synchronous
>> socket server - client model which works well.
>
> At www.freetds.org you'll find an asynchronous client. Not a minimal
> example, but the network functions are all restricted to one file.
>
> From experience I can tell you that network programming cannot be
> learned from online sources. You need books. I recommend
>
> The Linux Programming Interface by Michael Kerrisk
> UNIX Network Programming by W. Richard Stevens

I'd also suggest
Advanced Programming in the UNIX Environment
by W. Richard Stevens
which covers the "asyncronous non-blocking" part of the task

> If you can, read all Stevens' books. Worst case, you'll learn more
> than you need to know.

Agreed++

--
Lew Pitcher
"In Skills, We Trust"
PGP public key available upon request

[Mark]

+1

(Although I don't use non-blocking sockets any more. I prefer to
implement the code in separate threads).

--
<insert witty sig here>

[Jorgen Grahn]

On Mon, 2018-04-02, Mark wrote:
...

> (Although I don't use non-blocking sockets any more. I prefer to
> implement the code in separate threads).

Interesting; why? The trend has (it seems to me) gone in the opposite
direction in the last decade.

I have a feeling that implementing (say) a TCP server with threads
requires careful design and experience, just like non-blocking
sockets. I have only seen bad examples; do you have any good ones?

/Jorgen

[Rainer Weikusat]

Jorgen Grahn <grahn...@snipabacken.se> writes:
> On Mon, 2018-04-02, Mark wrote:
> ...
>> (Although I don't use non-blocking sockets any more. I prefer to
>> implement the code in separate threads).
>
> Interesting; why? The trend has (it seems to me) gone in the opposite
> direction in the last decade.
>
> I have a feeling that implementing (say) a TCP server with threads
> requires careful design and experience, just like non-blocking
> sockets. I have only seen bad examples; do you have any good ones?

A 'naive' multithreaded TCP-server isn't much different from a
traditional 'forking server', just the context-switch overhead is lower
due to not switching to a different addressspace and the implementation
is less robust because something like an invalid memory access in one
thread will terminate the complete server, not just the faulting
instance (serving one connection).

[Mark]

On 2 Apr 2018 18:50:50 GMT, Jorgen Grahn <grahn...@snipabacken.se>
wrote:

>On Mon, 2018-04-02, Mark wrote:
>...
>> (Although I don't use non-blocking sockets any more. I prefer to
>> implement the code in separate threads).
>
>Interesting; why?

I find it easier. Most of the applications I work on use multiple
threads anyway and it's not difficult to find a suitable framework.

>The trend has (it seems to me) gone in the opposite
>direction in the last decade.

Really? I am surprised.

>I have a feeling that implementing (say) a TCP server with threads
>requires careful design and experience, just like non-blocking
>sockets. I have only seen bad examples; do you have any good ones?

All programming requires careful design & experience. Unfortunately I
can't share any of the code I am working on now but I could write some
for you (at a cost ;-)

[Rainer Weikusat]

Mark <i...@dontgetlotsofspamanymore.net> writes:
> On 2 Apr 2018 18:50:50 GMT, Jorgen Grahn <grahn...@snipabacken.se>
> wrote:
>
>>On Mon, 2018-04-02, Mark wrote:
>>...
>>> (Although I don't use non-blocking sockets any more. I prefer to
>>> implement the code in separate threads).
>>
>>Interesting; why?
>
> I find it easier.

"Threads are great because this means I don't have to do state
machines!" rearing its ugly head once more ... sooner or later, this
will lead to the rediscovery of kernel stacks and then, we'll get
another round of "cooperative userspace threading" ...

[Nicolas George]

Mark , dans le message <js89cddudnbndmpbp...@4ax.com>, a
écrit :

> I find it easier. Most of the applications I work on use multiple
> threads anyway and it's not difficult to find a suitable framework.

I find that hard to believe: POSIX threads functions play very badly
with I/O operations. If you need even a stupid timeout, you need to go
back to using poll() and non-blocking fds. Or you need to hope
pthread_cancel() works as advertised, use another thread, and track all
code paths for pthread_setcancelstate().

[Mark]

On 04 Apr 2018 16:39:24 GMT, Nicolas George

<nicolas$geo...@salle-s.org> wrote:

>Mark , dans le message <js89cddudnbndmpbp...@4ax.com>, a
> écrit :
>> I find it easier. Most of the applications I work on use multiple
>> threads anyway and it's not difficult to find a suitable framework.
>
>I find that hard to believe: POSIX threads functions play very badly
>with I/O operations.

Cite? I have personal experience of large enterprise applications
that work fine like I propose.

>If you need even a stupid timeout, you need to go
>back to using poll() and non-blocking fds.

No you don't.

>Or you need to hope
>pthread_cancel() works as advertised, use another thread, and track all
>code paths for pthread_setcancelstate().

Again - you don't need to do it this way.

[Kaz Kylheku]

On 2018-04-04, Nicolas George <nicolas$geo...@salle-s.org> wrote:
> Mark , dans le message <js89cddudnbndmpbp...@4ax.com>, a
> écrit :
>> I find it easier. Most of the applications I work on use multiple
>> threads anyway and it's not difficult to find a suitable framework.
>
> I find that hard to believe: POSIX threads functions play very badly
> with I/O operations.

POSIX threads is a badly designed pile of crap which started as
DCE threads bolted onto a Unix process using user space hacks,
and hasn't changed the fundamental model.

It doesn't play along properly with any classic process global state:
chdir, setlocale, setenv.

It doesn't play along nicely with fork.

The integration of UNIX signals and pthreads is a mess.

I feel that in 2018, there is no reason to use these silly threads any
more and just fork processes. Heck, people are splitting their
applications into different virtual machine containers even.

Threading is just something with which programmers get infatuated when
they are in their professional puberty.

There was a boom in making everything threaded in the 1990's. SMP
machines broke into the consumer market, and the caches were still small
and CPU's slow that processes looked heavy weight.

I was really swept up in this crap myself, and ended up working on the
threading bits in Glibc, submitting patches to Drepper.

I had mutexes and condition variables working in the Linux kernel long
before Linux had these things.

http://www.kylheku.com/~kaz/lmc.html [1999]

I once made a poll-like multiplexing function (in the kernel) which
would atomically unlock a mutex, wait on a condition variable, **and**
poll multiple "struct file *" and "struct socket *" handles for
activity, all in one function call.

Hey, it turns out I added this to the above library!!! Holy crap, I
thought that piece code had been lost.

[Rainer Weikusat]

Kaz Kylheku <157-07...@kylheku.com> writes:
> On 2018-04-04, Nicolas George <nicolas$geo...@salle-s.org> wrote:
>> Mark , dans le message <js89cddudnbndmpbp...@4ax.com>, a
>> écrit :
>>> I find it easier. Most of the applications I work on use multiple
>>> threads anyway and it's not difficult to find a suitable framework.
>>
>> I find that hard to believe: POSIX threads functions play very badly
>> with I/O operations.
>
> POSIX threads

[...]

> The integration of UNIX signals and pthreads is a mess.

Hmm ... the integration of 'UNIX(*) signals' and the average
programmer's mind is a mess. I find them generally straight-forward to
use, even in multi-threaded applications.

The basic idea behind 'signal handling in POSIX threads' is that any
thread not currently blocking a signal is eligible to handle it. IMHO,
this makes perfect sense.

[bol...@cylonhq.com]

On Wed, 4 Apr 2018 18:14:37 +0000 (UTC)
Kaz Kylheku <157-07...@kylheku.com> wrote:
>I feel that in 2018, there is no reason to use these silly threads any
>more and just fork processes. Heck, people are splitting their
>applications into different virtual machine containers even.

I agree. Threads solve N problems but create N^2 problems in return. Unless
your application needs really tightly coupled multiple execution paths then
it makes far more sense to use multiprocess linked with pipes and/or SysV IPC.

Pity poor Windows programmers however - that hopeless kernel still can't do
a passable imitation of fork() and as for its signal handling...

[Geoff Clare]

Kaz Kylheku wrote:

> POSIX threads
[...]

> doesn't play along properly with any classic process global state:
> chdir,

Which is why openat() etc. were added to POSIX in 2008.

> setlocale,

Which is why uselocale() was added to POSIX in 2008.

> setenv.

True (but seldom needed?)

> It doesn't play along nicely with fork.

It's fine if all you're going to do in the child is call an exec*()
function (perhaps preceded by some simple atomic operations like
file descriptor redirections), but I agree there are some situations
where fork is a problem. It's unfortunate that the pthread_atfork()
"solution" turned out to be a dud.

--
Geoff Clare <net...@gclare.org.uk>

[Rainer Weikusat]

bol...@cylonHQ.com writes:
> On Wed, 4 Apr 2018 18:14:37 +0000 (UTC)
> Kaz Kylheku <157-07...@kylheku.com> wrote:
>>I feel that in 2018, there is no reason to use these silly threads any
>>more and just fork processes. Heck, people are splitting their
>>applications into different virtual machine containers even.
>
> I agree. Threads solve N problems but create N^2 problems in return.

As?

Threads provide an additional feature which isn't otherwise easily
available: Multiple threads of execution can work with shared state
information containing pointers. If used, this need to be implemented
such that some object residing in the shared address space isn't
modified by one thread while it's being accessed (both read-write and
read-only) by another thread.

The implementations I've encountered in practice are still vastly
simpler than equivalent code based on passing messages formatted
according to some protocol definition.

NB: I've used both approaches extensively myself.

[Rainer Weikusat]

Geoff Clare <ge...@clare.See-My-Signature.invalid> writes:
> Kaz Kylheku wrote:
>> POSIX threads

[...]

>> It doesn't play along nicely with fork.
>
> It's fine if all you're going to do in the child is call an exec*()
> function (perhaps preceded by some simple atomic operations like
> file descriptor redirections), but I agree there are some situations
> where fork is a problem. It's unfortunate that the pthread_atfork()
> "solution" turned out to be a dud.

I'm not buying this: Sensible semantics for combining threads and fork
were never designed because 'threadies' are as opposed to 'fork' as
'forkies' are opposed to threads.

In an ideal universe, this useless culture war from the past would die.
But that's unlikley for as long as combattants are still alive because
nobody's ever wrong in CS, he's always just a visionary whose time
hasn't yet come[*].

[*] N. Wirth still writes essays about 'properly looking assignment
operators' ...

[Scott Lurndal]

Rainer Weikusat <rwei...@talktalk.net> writes:
>Geoff Clare <ge...@clare.See-My-Signature.invalid> writes:
>> Kaz Kylheku wrote:
>>> POSIX threads
>
>[...]
>
>>> It doesn't play along nicely with fork.
>>
>> It's fine if all you're going to do in the child is call an exec*()
>> function (perhaps preceded by some simple atomic operations like
>> file descriptor redirections), but I agree there are some situations
>> where fork is a problem. It's unfortunate that the pthread_atfork()
>> "solution" turned out to be a dud.
>
>I'm not buying this: Sensible semantics for combining threads and fork
>were never designed because 'threadies' are as opposed to 'fork' as
>'forkies' are opposed to threads.

That, while facile, is false.

Note that forkall() was part of Unixware.

[Kaz Kylheku]

On 2018-04-05, Rainer Weikusat <rwei...@talktalk.net> wrote:
> Geoff Clare <ge...@clare.See-My-Signature.invalid> writes:
>> Kaz Kylheku wrote:
>>> POSIX threads
>
> [...]
>
>>> It doesn't play along nicely with fork.
>>
>> It's fine if all you're going to do in the child is call an exec*()
>> function (perhaps preceded by some simple atomic operations like
>> file descriptor redirections), but I agree there are some situations
>> where fork is a problem. It's unfortunate that the pthread_atfork()
>> "solution" turned out to be a dud.
>
> I'm not buying this: Sensible semantics for combining threads and fork
> were never designed because 'threadies' are as opposed to 'fork' as
> 'forkies' are opposed to threads.

As long as an application written by forkies never has to use a
third-party library (or, worse, dynamic plugin beyond their control)
written by threadies, and vice versa, everything is cool.

[Rainer Weikusat]

AFAIK, it wasn't ever part of pthreads. It's also just the other
"non-solution", cf

For example, when one thread reads a file serially and another
thread in the process successfully calls one of the forks, each
process then contains a thread that is reading the file. Because
the seek pointer for a file descriptor is shared after a fork(),
the thread in the parent gets some data while the thread in the
child gets the other.

https://docs.oracle.com/cd/E19455-01/806-5257/gen-6/index.html

As I've written here at least once in the past, a sensible marriage for
fork and multi-threading should provide "fork-blocking" facilities
similar to those which exist for signals.

[Rainer Weikusat]

As soon as an application loads abitrary machine code provided by a
third party into its addressspace and actually executes that, all bets
are off. This is a strawman.

[Scott Lurndal]

That's a bit of a red-herring. In fact, in a properly written
multithreaded application, one wouldn't call fork (or forkall) sans exec
unless one had placed the reset of the application in a state where
fork()/forkall() would not have unusual side effects. A library that provides
an api which uses threads internally should include a 'prepare_for_fork'
and 'post_fork' API that can be used by the library client.

As it happens, in all the multithreaded application-leven code I've written or
worked on in the last 20 years, not one has needed to fork (sans immediate exec)
for any reason.

[bol...@cylonhq.com]

On Thu, 05 Apr 2018 16:16:25 +0100
Rainer Weikusat <rwei...@talktalk.net> wrote:
>bol...@cylonHQ.com writes:
>> On Wed, 4 Apr 2018 18:14:37 +0000 (UTC)
>> Kaz Kylheku <157-07...@kylheku.com> wrote:
>>>I feel that in 2018, there is no reason to use these silly threads any
>>>more and just fork processes. Heck, people are splitting their
>>>applications into different virtual machine containers even.
>>
>> I agree. Threads solve N problems but create N^2 problems in return.
>
>As?

Race conditions, deadlocks, non thread safe library functions, non repeatable
crashes etc.

>Threads provide an additional feature which isn't otherwise easily
>available: Multiple threads of execution can work with shared state
>information containing pointers. If used, this need to be implemented

What do you think shared memory is?

>such that some object residing in the shared address space isn't
>modified by one thread while it's being accessed (both read-write and
>read-only) by another thread.

pthreads allows you to create a mutex in shared memory.

Google pthread_mutexattr_setpshared()

>The implementations I've encountered in practice are still vastly
>simpler than equivalent code based on passing messages formatted
>according to some protocol definition.

A simple threaded program is usually one in which the author hasn't considered
all code paths and at some point it will crash or lock up because of threading
issues.

[Scott Lurndal]

bol...@cylonHQ.com writes:
>On Thu, 05 Apr 2018 16:16:25 +0100
>Rainer Weikusat <rwei...@talktalk.net> wrote:

>>Threads provide an additional feature which isn't otherwise easily
>>available: Multiple threads of execution can work with shared state
>>information containing pointers. If used, this need to be implemented
>
>What do you think shared memory is?

Very inconvenient to use when compared with threads and pthread_mutex,
pthread_cond.

[bol...@cylonhq.com]

Whats inconvenient about using pointers and mapped structures? And if you
think condition variables are simple to use then I'll have some of what you're
smoking!

[Scott Lurndal]

Yes, they're easy to use. I've been using similar primitives since
1985 (http://vseries.lurndal.org/doku.php?id=instructions:lok).

They're far easier to use and reason about than system V semaphores,
for example, which is one of the few ways to synchronize multiple
processes sharing memory.

Note that neither mmap nor shmat guarantee the address at which
the segment is mapped, and MAP_FIXED isn't guaranteed to map at
the requested address, so one cannot just use pointers, one must
instead use offsets from the start of the mapping.

[Rainer Weikusat]

bol...@cylonHQ.com writes:
> On Thu, 05 Apr 2018 16:16:25 +0100
> Rainer Weikusat <rwei...@talktalk.net> wrote:
>>bol...@cylonHQ.com writes:
>>> On Wed, 4 Apr 2018 18:14:37 +0000 (UTC)
>>> Kaz Kylheku <157-07...@kylheku.com> wrote:
>>>>I feel that in 2018, there is no reason to use these silly threads any
>>>>more and just fork processes. Heck, people are splitting their
>>>>applications into different virtual machine containers even.
>>>
>>> I agree. Threads solve N problems but create N^2 problems in return.
>>
>>As?
>
> Race conditions, deadlocks, non thread safe library functions, non repeatable
> crashes etc.

Race conditions and deadlocks are errors which occur because of
incorrectly implemented inter-thread communication, regardless of these
different threads running in one or in several address spaces.

Crashes are always 'repeatable' unless caused they're caused by
transient hardware errors (even then, they're technically repeatable,
just not necessarily controlled by software alone). But if the
conditions enabling to repeat them were known prior to debugging the
issue, it shouldn't have occurred in the first place.

Even in absence of writes through entirely wild pointers, 'interesting'
crashes occur because of some n-stage chaining of more or less rarely
occuring, external events, eg, timing of messages received over the
network (oops ... the reast of the univers runs asynchronously in
relation to 'us' and it can cause stuff in 'our' address space ...).

>>Threads provide an additional feature which isn't otherwise easily
>>available: Multiple threads of execution can work with shared state
>>information containing pointers. If used, this need to be implemented
>
> What do you think shared memory is?

There's no way to have shared pointers in shared memory unless all
applications using manage their own addressspace at least to the degree
that they can do mmap with MAP_FIXED for some common addresss.

>>such that some object residing in the shared address space isn't
>>modified by one thread while it's being accessed (both read-write and
>>read-only) by another thread.
>
> pthreads allows you to create a mutex in shared memory.
>
> Google pthread_mutexattr_setpshared()
>
>>The implementations I've encountered in practice are still vastly
>>simpler than equivalent code based on passing messages formatted
>>according to some protocol definition.
>
> A simple threaded program is usually one in which the author hasn't considered
> all code paths and at some point it will crash or lock up because of threading
> issues.

Any program is usually one where not all codepaths have been tested and
which will sooner or later crash because of this. Or go into an endless
loop.

[Kaz Kylheku]

On 2018-04-06, Scott Lurndal <sc...@slp53.sl.home> wrote:
> Note that neither mmap nor shmat guarantee the address at which
> the segment is mapped

Not true if you allocate the mmap in the parent, then fork! :)

[bol...@cylonhq.com]

On Fri, 06 Apr 2018 14:40:33 GMT

sc...@slp53.sl.home (Scott Lurndal) wrote:
>bol...@cylonHQ.com writes:
>>On Fri, 06 Apr 2018 13:19:44 GMT
>>sc...@slp53.sl.home (Scott Lurndal) wrote:
>>>bol...@cylonHQ.com writes:
>>>>On Thu, 05 Apr 2018 16:16:25 +0100
>>>>Rainer Weikusat <rwei...@talktalk.net> wrote:
>>>
>>>>>Threads provide an additional feature which isn't otherwise easily
>>>>>available: Multiple threads of execution can work with shared state
>>>>>information containing pointers. If used, this need to be implemented
>>>>
>>>>What do you think shared memory is?
>>>
>>>Very inconvenient to use when compared with threads and pthread_mutex,
>>>pthread_cond.
>>
>>Whats inconvenient about using pointers and mapped structures? And if you
>>think condition variables are simple to use then I'll have some of what you're
>>smoking!
>
>Yes, they're easy to use. I've been using similar primitives since
>1985 (http://vseries.lurndal.org/doku.php?id=instructions:lok).

Good for you. I find them an utter PITA along with a lot of the pthreads
library in general.

>They're far easier to use and reason about than system V semaphores,
>for example, which is one of the few ways to synchronize multiple
>processes sharing memory.

How many ways do you want? You can use semaphores, messages, mutexes, signals,
pipes, sockets and probably some others I've forgotten about.

>Note that neither mmap nor shmat guarantee the address at which
>the segment is mapped, and MAP_FIXED isn't guaranteed to map at
>the requested address, so one cannot just use pointers, one must
>instead use offsets from the start of the mapping.

Err yes, exactly, you use pointers to the shared memory. Thats the whole
point of it.

[Kaz Kylheku]

On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:
> bol...@cylonHQ.com writes:
>> On Thu, 05 Apr 2018 16:16:25 +0100
>> Rainer Weikusat <rwei...@talktalk.net> wrote:
>>>bol...@cylonHQ.com writes:
>>>> On Wed, 4 Apr 2018 18:14:37 +0000 (UTC)
>>>> Kaz Kylheku <157-07...@kylheku.com> wrote:
>>>>>I feel that in 2018, there is no reason to use these silly threads any
>>>>>more and just fork processes. Heck, people are splitting their
>>>>>applications into different virtual machine containers even.
>>>>
>>>> I agree. Threads solve N problems but create N^2 problems in return.
>>>
>>>As?
>>
>> Race conditions, deadlocks, non thread safe library functions, non repeatable
>> crashes etc.
>
> Race conditions and deadlocks are errors which occur because of
> incorrectly implemented inter-thread communication, regardless of these
> different threads running in one or in several address spaces.
>
> Crashes are always 'repeatable' unless caused they're caused by

Nope! If we take some program that does exactly the same thing
on the same input test case, like a compiler or whatever,
and make it threaded, then its behavior is no longer the same
on every run due to the randomization of execution order by the
scheduler.

(And, unfortunately, idiotic non-thread-related nonsense like address
space randomization.)

[Rainer Weikusat]

This works only if the maximum amount of memory which will be needed is
known in advance. Further, it needs a custom memory allocator. And then,
this just means you've now introduced all so-called "threading problems"
into your forking application as you have multiple threads of execution
working in a shared address space (segment) with the exact same
requirements wrt "not stepping onto each others toes".

IOW, you get all the drawbacks and none of the benefits.

Bargain!

[bol...@cylonhq.com]

On Fri, 06 Apr 2018 15:56:57 +0100

Rainer Weikusat <rwei...@talktalk.net> wrote:
>bol...@cylonHQ.com writes:
>> On Thu, 05 Apr 2018 16:16:25 +0100
>> Rainer Weikusat <rwei...@talktalk.net> wrote:
>>>bol...@cylonHQ.com writes:
>>>> On Wed, 4 Apr 2018 18:14:37 +0000 (UTC)
>>>> Kaz Kylheku <157-07...@kylheku.com> wrote:
>>>>>I feel that in 2018, there is no reason to use these silly threads any
>>>>>more and just fork processes. Heck, people are splitting their
>>>>>applications into different virtual machine containers even.
>>>>
>>>> I agree. Threads solve N problems but create N^2 problems in return.
>>>
>>>As?
>>
>> Race conditions, deadlocks, non thread safe library functions, non repeatable
>> crashes etc.
>
>Race conditions and deadlocks are errors which occur because of
>incorrectly implemented inter-thread communication, regardless of these
>different threads running in one or in several address spaces.

Threaded code is essentially parallel code and writing parallel code of any
reasonable sophistication is hard. Anyone who says otherwise either doesn't
understand the problems involved or is a genius.

>Crashes are always 'repeatable' unless caused they're caused by
>transient hardware errors (even then, they're technically repeatable,

Rubbish. You have no way to control when a particular thread will execute
unless you manually block it, and if a crash is caused by a race condition when
the threads are in a particular state it may take many goes to recreate it.

>Even in absence of writes through entirely wild pointers, 'interesting'
>crashes occur because of some n-stage chaining of more or less rarely
>occuring, external events, eg, timing of messages received over the
>network (oops ... the reast of the univers runs asynchronously in
>relation to 'us' and it can cause stuff in 'our' address space ...).

You just contradicted yourself.

>There's no way to have shared pointers in shared memory unless all
>applications using manage their own addressspace at least to the degree
>that they can do mmap with MAP_FIXED for some common addresss.

Huh? Shared memory is shared, of course you can have shared pointers to it.
If you couldn't there wouldn't be any point to it! The hard part isn't
sharing the memory, its providing safe access to it but as I've already said,
you can use mutexes in shared memory too, you don't have to rely on SysV
semaphores or messages which frankly IMO are awful.

>> A simple threaded program is usually one in which the author hasn't
>considered
>> all code paths and at some point it will crash or lock up because of
>threading
>> issues.
>
>Any program is usually one where not all codepaths have been tested and
>which will sooner or later crash because of this. Or go into an endless
>loop.

Threading makes that situation exponentially more likely.

[Rainer Weikusat]

This makes the crash no less repeatable: If the same events happen in
the same order, the outcome will be the same.

[James Kuyper]

On 04/06/2018 11:19 AM, Rainer Weikusat wrote:
> Kaz Kylheku <157-07...@kylheku.com> writes:
>> On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:

...

>>> Crashes are always 'repeatable' unless caused they're caused by
>>
>> Nope! If we take some program that does exactly the same thing
>> on the same input test case, like a compiler or whatever,
>> and make it threaded, then its behavior is no longer the same
>> on every run due to the randomization of execution order by the
>> scheduler.
>
> This makes the crash no less repeatable: If the same events happen in
> the same order, the outcome will be the same.

I've always taken the term "repeatable" to imply, at least in this kind
of context, that it's not only feasible, but easy, to arrange that the
same events happen in the same order. As far as I can see, in the
context he's describing, it's simply not feasible to arrange that. It
might happen by accident, but relying on a convenient miracle is not
what I consider "repeatable".

[Rainer Weikusat]

James Kuyper <james...@verizon.net> writes:
> On 04/06/2018 11:19 AM, Rainer Weikusat wrote:
>> Kaz Kylheku <157-07...@kylheku.com> writes:
>>> On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:
> ...
>>>> Crashes are always 'repeatable' unless caused they're caused by
>>>
>>> Nope! If we take some program that does exactly the same thing
>>> on the same input test case, like a compiler or whatever,
>>> and make it threaded, then its behavior is no longer the same
>>> on every run due to the randomization of execution order by the
>>> scheduler.
>>
>> This makes the crash no less repeatable: If the same events happen in
>> the same order, the outcome will be the same.
>
> I've always taken the term "repeatable" to imply, at least in this kind
> of context, that it's not only feasible, but easy, to arrange that the
> same events happen in the same order.

That's certainly a bonus: Multi-threaded application, isolated
application with races, easy, single-threaded, networked application
relying on 'unpredictiable' client behaviour and message timing - hard.

Regardless of ideological bollocks, that's a practical reality I'm all
to familiar with.

[James Kuyper]

On 04/06/2018 11:18 AM, bol...@cylonHQ.com wrote:
> On Fri, 06 Apr 2018 15:56:57 +0100
> Rainer Weikusat <rwei...@talktalk.net> wrote:

...

>> There's no way to have shared pointers in shared memory unless all
>> applications using manage their own addressspace at least to the degree
>> that they can do mmap with MAP_FIXED for some common addresss.
>
> Huh? Shared memory is shared, of course you can have shared pointers to it.
> If you couldn't there wouldn't be any point to it!

He's describing the case where the same block of memory is shared at
different addresses by different processes. Therefore a pointer into a
part of that memory created in one process would have a different
address than a pointer pointing at the same location as seen by another
process. How could such a pointer value be usefully shared between the
processes? It could be passed from process 1 to process 2, but the only
useful thing that process 2 could do with such a pointer would be to
pass it back to process 1, where that address it contains is the correct
address of the pointed-at location. I wouldn't call such a pointer a
shared pointer.

If all processes sharing the memory use MAP_FIXED and specify the exact
same address, then sharing pointer values between processes would be
feasible - but such code is not portable. Otherwise, the best you can do
is calculate offsets from the beginning of the mapped mermoy

[Rainer Weikusat]

Addition: That's obviously not exclusively true. Repeating/ fixing crashes in all
kinds of applications can be (relatively) easy or (relatively) hard,
it's just that crashes caused by multiple threads racing with each other
aren't necessarily difficult to diagnose and fix just because of
"multiple threads racing with each other".

NB: I've two specific issues in mind here, one occuring in a
single-threaded application on whose code I've been working on since
2010 and another occuring in a multi-threaded on I didn't ever see since
last year and to which I only 'cursorily' added a few simple features.

Fixing the second required nothing but reading the involved code to work
out "what could go wrong here", the first took me about half a week of
headscratching and studying diagnostic logs (impossible to reproduce for
me), including moments of "it must be bewitched!" desparation, before I
had worked out the chain of events.

[Kaz Kylheku]

On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:

> Kaz Kylheku <157-07...@kylheku.com> writes:
>> On 2018-04-06, Scott Lurndal <sc...@slp53.sl.home> wrote:
>>> Note that neither mmap nor shmat guarantee the address at which
>>> the segment is mapped
>>
>> Not true if you allocate the mmap in the parent, then fork! :)
>
> This works only if the maximum amount of memory which will be needed is
> known in advance.

I think these things can be set to grow lazily.

That's what is done with your brk anyway and the stack on the opposite
end.

So that is say, only the maximum amount of virtual space need be laid
out, not actual memory.

> Further, it needs a custom memory allocator. And then,

There are off-the-shelf shared mem allocators.

> this just means you've now introduced all so-called "threading problems"
> into your forking application as you have multiple threads of execution
> working in a shared address space (segment) with the exact same
> requirements wrt "not stepping onto each others toes".

Nope; not the same requirements. They have their own signals, their own
working directory, their own stdin/stdout, their own environment
variables. They can use nftw for walking directories.

Their own everything that is not in that shared memory.

Calling a third party library that happens to use strtok presents no problem.

[Kaz Kylheku]

"Repeatable" is unconditional; you're talking about "reproducible".

Like whatever situation was observed is logged in the bug database as
"5% reproducible".

[Kaz Kylheku]

On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:
> James Kuyper <james...@verizon.net> writes:
>> On 04/06/2018 11:19 AM, Rainer Weikusat wrote:
>>> Kaz Kylheku <157-07...@kylheku.com> writes:
>>>> On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:
>> ...
>>>>> Crashes are always 'repeatable' unless caused they're caused by
>>>>
>>>> Nope! If we take some program that does exactly the same thing
>>>> on the same input test case, like a compiler or whatever,
>>>> and make it threaded, then its behavior is no longer the same
>>>> on every run due to the randomization of execution order by the
>>>> scheduler.
>>>
>>> This makes the crash no less repeatable: If the same events happen in
>>> the same order, the outcome will be the same.
>>
>> I've always taken the term "repeatable" to imply, at least in this kind
>> of context, that it's not only feasible, but easy, to arrange that the
>> same events happen in the same order.
>
> That's certainly a bonus: Multi-threaded application, isolated
> application with races, easy, single-threaded, networked application
> relying on 'unpredictiable' client behaviour and message timing - hard.

Taking a program which does things step by step in an orderly way with a
repeatable result, and scrambling those steps so that it sometimes
doesn't work, is not a "bonus".

The original program is not broken.

It's an extraordinary tremendous risk that requires an extraordinary
benefit to justify it.

[Rainer Weikusat]

"5% reproducible" is birtspeak for "we have no clue why this happens,
but we consider it unimportant nevertheless". *If* the ordered sequence
of events causing the phenomenon was known, it would be "100%
reproducible" (subject to being able to recreate the sequence).

Example I've been alluding to so far: Program which sends an e-mail via
libcurl constructed from results of various (SQLite) SQL
queries. Libcurl invokes a callback as soon as it wants data to send to
the remote SMTP-server. In the given case, one thread would allocate a
"message queue" structure, send that to another thread over a message
queue and then call the top-level libcurl entry point with a pointer to
a callback which would read the data from the message queue send to the
second thread. After the top-level curl call returned, the message queue
sent to the second thread would be freed by the first.

The second thread would perform the necessary queries and send the data
over the message queue sent to it.

Problem: If libcurl couldn't successfully connect to the SMTP-server,
the top-level call would return without ever invoking the
callback. Because of this, the first thread wouldn't ever wait for the
second to do anything, hence, the second would sometimes cause a crash
because the first thread freed the message queue before the second was
done using it.

Perfectly "repeatable" by ensuring the second thread would never be done
using the message queue before the first freed it by causing the 2nd to
sleep for a few seconds before doing anything.

NB: I'm not responsible for this 'interesting' (lightly put) design.

[Rainer Weikusat]

Kaz Kylheku <157-07...@kylheku.com> writes:
> On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:
>> Kaz Kylheku <157-07...@kylheku.com> writes:
>>> On 2018-04-06, Scott Lurndal <sc...@slp53.sl.home> wrote:
>>>> Note that neither mmap nor shmat guarantee the address at which
>>>> the segment is mapped
>>>
>>> Not true if you allocate the mmap in the parent, then fork! :)
>>
>> This works only if the maximum amount of memory which will be needed is
>> known in advance.
>
> I think these things can be set to grow lazily.
>
> That's what is done with your brk anyway and the stack on the opposite
> end.
>
> So that is say, only the maximum amount of virtual space need be laid
> out, not actual memory.

The "maximum amount of virtual space" is exactly the same as "the
maximum amount of memory" from the viewpoint of an application.

>> Further, it needs a custom memory allocator. And then,
>
> There are off-the-shelf shared mem allocators.

"But I can download that from the internet!" still means "it needs a
custom memory allocator".

>> this just means you've now introduced all so-called "threading problems"
>> into your forking application as you have multiple threads of execution
>> working in a shared address space (segment) with the exact same
>> requirements wrt "not stepping onto each others toes".
>
> Nope; not the same requirements. They have their own signals, their own
> working directory, their own stdin/stdout, their own environment
> variables. They can use nftw for walking directories.
>
> Their own everything that is not in that shared memory.

And for anything using "this shared memory" (I mentioned that
explicitly), everything said about race conditions, deadlocks and
"non-deterministc behaviour" applies just as if all threads were part of
a single process.

> Calling a third party library that happens to use strtok presents no
> problem.

But calling a third party library calling malloc does. Not to mention
that "calling a third party library that happens to use strtok"
"presents problems" unless the application doesn't use strtok or uses it
only such that this use doesn't conflict with the library's use.

[Kaz Kylheku]

On 2018-04-07, Rainer Weikusat <rwei...@talktalk.net> wrote:
> Kaz Kylheku <157-07...@kylheku.com> writes:
>> On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:
>>> Kaz Kylheku <157-07...@kylheku.com> writes:
>>>> On 2018-04-06, Scott Lurndal <sc...@slp53.sl.home> wrote:
>>>>> Note that neither mmap nor shmat guarantee the address at which
>>>>> the segment is mapped
>>>>
>>>> Not true if you allocate the mmap in the parent, then fork! :)
>>>
>>> This works only if the maximum amount of memory which will be needed is
>>> known in advance.
>>
>> I think these things can be set to grow lazily.
>>
>> That's what is done with your brk anyway and the stack on the opposite
>> end.
>>
>> So that is say, only the maximum amount of virtual space need be laid
>> out, not actual memory.
>
> The "maximum amount of virtual space" is exactly the same as "the
> maximum amount of memory" from the viewpoint of an application.

Program loading is already based on a maximum, predetermined space.

E.g. there is a "brk" fence on one end which is moved to make
allocations, a downward growing main stack on the other end, emanating
from some fixed address, and an arena in between for memory mappings.

That's all a big space which is pre-determined in advance, same for
every program, and only populated with pages later.

>>> Further, it needs a custom memory allocator. And then,
>>
>> There are off-the-shelf shared mem allocators.
>
> "But I can download that from the internet!" still means "it needs a
> custom memory allocator".

glibc is downloaded from the Internet, so malloc on Linux is a custom
allocator.

If there is a "factory" one, it's the in Windows that was uninstalled
from the PC. :)

[Rainer Weikusat]

Kaz Kylheku <157-07...@kylheku.com> writes:
> On 2018-04-07, Rainer Weikusat <rwei...@talktalk.net> wrote:
>> Kaz Kylheku <157-07...@kylheku.com> writes:
>>> On 2018-04-06, Rainer Weikusat <rwei...@talktalk.net> wrote:
>>>> Kaz Kylheku <157-07...@kylheku.com> writes:
>>>>> On 2018-04-06, Scott Lurndal <sc...@slp53.sl.home> wrote:
>>>>>> Note that neither mmap nor shmat guarantee the address at which
>>>>>> the segment is mapped
>>>>>
>>>>> Not true if you allocate the mmap in the parent, then fork! :)
>>>>
>>>> This works only if the maximum amount of memory which will be needed is
>>>> known in advance.
>>>
>>> I think these things can be set to grow lazily.
>>>
>>> That's what is done with your brk anyway and the stack on the opposite
>>> end.
>>>
>>> So that is say, only the maximum amount of virtual space need be laid
>>> out, not actual memory.
>>
>> The "maximum amount of virtual space" is exactly the same as "the
>> maximum amount of memory" from the viewpoint of an application.
>
> Program loading is already based on a maximum, predetermined space.
>
> E.g. there is a "brk" fence on one end which is moved to make
> allocations, a downward growing main stack on the other end, emanating
> from some fixed address, and an arena in between for memory mappings.
>
> That's all a big space which is pre-determined in advance, same for
> every program, and only populated with pages later.

"Once upon a time in the past", I had a German teacher who was a total
dickhead and particularly 'fond' of me. He asked for 'the remarkable
difference' between a Schiller drama called "Die Raeuber" and earlier/
classical dramatic works. I happened to know the answer, namely, it was
more organized like a present-day movie where a varying cast of
characters enacts different subplots of some larger plot in a variety of
locations as opposed to Greek dramas which were (according to Aristotle)
supposed to happen in one place where one set of characters travel through
a single plot ("Einheit von Ort, Zeit und Handlung"). However, the guy
wouldn't let me answer it because as soon as I used a adjective
indicating a difference between the drama in question and 'classically
structured' plays, he would force me into an epistomological quarrell
over the exact meaning of the adjective I had just used instead of
allowing me to proceed to describing the difference. This went on, to
everybody's but mine's great amusement, for 15 - 20 minutes until I gave
up in despair[*].

You're certainly both intelligent and knowledgable enough to understand
what I meant to express, hence, I see no reason to engage in this mock
fight based on unavoidable ambiguities in natural languages.

[*] He actually managed to get my German grade down from the usual
B-and-occasional-A to D minus. OTOH, by that time, my math grades where
up from the usual D minus to A minus because the math teachers I had
weren't fighting me in the name of "all that is good, just and
proper".

[bol...@cylonhq.com]

On Fri, 6 Apr 2018 12:04:26 -0400
James Kuyper <james...@verizon.net> wrote:
>On 04/06/2018 11:18 AM, bol...@cylonHQ.com wrote:
>> On Fri, 06 Apr 2018 15:56:57 +0100
>> Rainer Weikusat <rwei...@talktalk.net> wrote:

>....

>>> There's no way to have shared pointers in shared memory unless all
>>> applications using manage their own addressspace at least to the degree
>>> that they can do mmap with MAP_FIXED for some common addresss.
>>
>> Huh? Shared memory is shared, of course you can have shared pointers to it.
>> If you couldn't there wouldn't be any point to it!
>
>He's describing the case where the same block of memory is shared at
>different addresses by different processes. Therefore a pointer into a
>part of that memory created in one process would have a different
>address than a pointer pointing at the same location as seen by another
>process. How could such a pointer value be usefully shared between the
>processes? It could be passed from process 1 to process 2, but the only

We're discussing threading vs multiprocess , ie fork(). So the shared memory
would be created at startup and then would be shared by all child processes
therefor the pointer addresses will be identical. It would be a pretty badly
designed shared memory system if the pointer returned by shmat() became
invalid in the child process immediately upon invocation of fork()!

>feasible - but such code is not portable. Otherwise, the best you can do
>is calculate offsets from the beginning of the mapped mermoy

Given thats basic arithmetic I don't see how that would even be an issue.

[Rainer Weikusat]

bol...@cylonHQ.com writes:
> On Fri, 6 Apr 2018 12:04:26 -0400
> James Kuyper <james...@verizon.net> wrote:
>>On 04/06/2018 11:18 AM, bol...@cylonHQ.com wrote:
>>> On Fri, 06 Apr 2018 15:56:57 +0100
>>> Rainer Weikusat <rwei...@talktalk.net> wrote:
>>....
>>>> There's no way to have shared pointers in shared memory unless all
>>>> applications using manage their own addressspace at least to the degree
>>>> that they can do mmap with MAP_FIXED for some common addresss.
>>>
>>> Huh? Shared memory is shared, of course you can have shared pointers to it.
>>> If you couldn't there wouldn't be any point to it!
>>
>>He's describing the case where the same block of memory is shared at
>>different addresses by different processes. Therefore a pointer into a
>>part of that memory created in one process would have a different
>>address than a pointer pointing at the same location as seen by another
>>process. How could such a pointer value be usefully shared between the
>>processes? It could be passed from process 1 to process 2, but the only
>
> We're discussing threading vs multiprocess , ie fork(). So the shared memory
> would be created at startup and then would be shared by all child processes
> therefor the pointer addresses will be identical.

That's one way to use shared memory segments. More generally, shared
memory exists independently of processes and unrelated processes can use
a common shared memory segment. *If* pointers in shared memory segments
are supposed to be usable by all processes using it, they at least all
need to map it at the same address. Stronger requirements, like "they
are all descendants of the process which created the [fixed-size] shared memory
segement" will obviously work, too.

[...]

>>feasible - but such code is not portable. Otherwise, the best you can do
>>is calculate offsets from the beginning of the mapped mermoy
>
> Given thats basic arithmetic I don't see how that would even be an issue.

It means that all accesses to something in the shared memory segment
need an additional indirection/ redirection step to acquire an address
that's valid now (the segment could have been grown and reattached at a
different address since the last access to it) and that all data in the
shared memory segment can't contain pointers, only offsets relative to
whatever its present start address happens to be, which means an
additional calculation is necessary for internal references, too.

This can obviously be implemented but it's more code and more 'internal
translation layers', hence, using such a model is less easy than using a
'naturally existing' shared address space some set of threads happens to
share.

[James Kuyper]

On 04/09/2018 04:37 AM, bol...@cylonHQ.com wrote:
> On Fri, 6 Apr 2018 12:04:26 -0400
> James Kuyper <james...@verizon.net> wrote:
>> On 04/06/2018 11:18 AM, bol...@cylonHQ.com wrote:
>>> On Fri, 06 Apr 2018 15:56:57 +0100
>>> Rainer Weikusat <rwei...@talktalk.net> wrote:
>> ....
>>>> There's no way to have shared pointers in shared memory unless all
>>>> applications using manage their own addressspace at least to the degree
>>>> that they can do mmap with MAP_FIXED for some common addresss.
>>>
>>> Huh? Shared memory is shared, of course you can have shared pointers to it.
>>> If you couldn't there wouldn't be any point to it!
>>
>> He's describing the case where the same block of memory is shared at
>> different addresses by different processes. Therefore a pointer into a
>> part of that memory created in one process would have a different
>> address than a pointer pointing at the same location as seen by another
>> process. How could such a pointer value be usefully shared between the
>> processes? It could be passed from process 1 to process 2, but the only
>
> We're discussing threading vs multiprocess , ie fork(). So the shared memory
> would be created at startup and then would be shared by all child processes
> therefor the pointer addresses will be identical. It would be a pretty badly
> designed shared memory system if the pointer returned by shmat() became
> invalid in the child process immediately upon invocation of fork()!

All that's required for multi-processing is multiple processes - they
don't have to be related to each other through fork()ing. mmap() can be
called by two different processes without either of them having been
created by calling fork() from the other. Look at the clause in his
statement starting with the word "unless" - calling mmap() before
creating the second process by fork()ing qualifies as "manag[ing] their
own address space" in the required sense. The problem he's talking about
would still be a problem even if fork() were used, if mmap() were called
separately in each process AFTER calling fork().

>> feasible - but such code is not portable. Otherwise, the best you can do
>> is calculate offsets from the beginning of the mapped mermoy
>
> Given thats basic arithmetic I don't see how that would even be an issue.

It's extra code - you have to constantly convert back and forth between
pointers and offsets from the base address of the mapped memory.

[bol...@cylonhq.com]

On Mon, 9 Apr 2018 08:36:42 -0400
James Kuyper <james...@verizon.net> wrote:
>On 04/09/2018 04:37 AM, bol...@cylonHQ.com wrote:
>> We're discussing threading vs multiprocess , ie fork(). So the shared memory
>> would be created at startup and then would be shared by all child processes
>> therefor the pointer addresses will be identical. It would be a pretty badly
>> designed shared memory system if the pointer returned by shmat() became
>> invalid in the child process immediately upon invocation of fork()!
>
>All that's required for multi-processing is multiple processes - they
>don't have to be related to each other through fork()ing. mmap() can be

No, but as I said, this is comparing with threads and threads are all
related to each other therefor a sane comparison is processes that are related
to each other.

>statement starting with the word "unless" - calling mmap() before
>creating the second process by fork()ing qualifies as "manag[ing] their
>own address space" in the required sense. The problem he's talking about
>would still be a problem even if fork() were used, if mmap() were called
>separately in each process AFTER calling fork().

Calling mmap seperately defeats the whole point of sharing it between related
processes. You're simply creating straw men.

>> Given thats basic arithmetic I don't see how that would even be an issue.
>
>It's extra code - you have to constantly convert back and forth between
>pointers and offsets from the base address of the mapped memory.

Big deal. Most pointer code involves some sort of pointer arithmetic anyway.

[james...@verizon.net]

On Monday, April 9, 2018 at 9:40:21 AM UTC-4, bol...@cylonhq.com wrote:
> On Mon, 9 Apr 2018 08:36:42 -0400
> James Kuyper <james...@verizon.net> wrote:
> >On 04/09/2018 04:37 AM, bol...@cylonHQ.com wrote:
> >> We're discussing threading vs multiprocess , ie fork(). So the shared memory
> >> would be created at startup and then would be shared by all child processes
> >> therefor the pointer addresses will be identical. It would be a pretty badly
> >> designed shared memory system if the pointer returned by shmat() became
> >> invalid in the child process immediately upon invocation of fork()!
> >
> >All that's required for multi-processing is multiple processes - they
> >don't have to be related to each other through fork()ing. mmap() can be
>
> No, but as I said, this is comparing with threads and threads are all
> related to each other therefor a sane comparison is processes that are related
> to each other.

From his first statement on the issue, it's been clear he was comparing threaded code with IPC between processes that are not related to each other by fork()ing. You might have no interest in such comparisons, considering them insane, but pretending he was talking about something else doesn't render your comments relevant to what he was actually talking about.

> >statement starting with the word "unless" - calling mmap() before
> >creating the second process by fork()ing qualifies as "manag[ing] their
> >own address space" in the required sense. The problem he's talking about
> >would still be a problem even if fork() were used, if mmap() were called
> >separately in each process AFTER calling fork().
>
> Calling mmap seperately defeats the whole point of sharing it between related
> processes. You're simply creating straw men.

I thought the purpose was to share memory? Processes that aren't related by fork()ing can still have reasons why it would be desirable for them to share memory.

> >> Given thats basic arithmetic I don't see how that would even be an issue.
> >
> >It's extra code - you have to constantly convert back and forth between
> >pointers and offsets from the base address of the mapped memory.
>
> Big deal. Most pointer code involves some sort of pointer arithmetic anyway.

So you don't mind having to replace every use of *p with *(T*)(map_base + p)? It would certainly bother me.

[bol...@cylonhq.com]

On Mon, 9 Apr 2018 07:09:51 -0700 (PDT)
james...@verizon.net wrote:
>On Monday, April 9, 2018 at 9:40:21 AM UTC-4, bol...@cylonhq.com wrote:
>> No, but as I said, this is comparing with threads and threads are all

>> related to each other therefor a sane comparison is processes that are re=
>lated
>> to each other.
>
>From his first statement on the issue, it's been clear he was comparing thr=
>eaded code with IPC between processes that are not related to each other by=

Which is a pointless comparison. Apples and oranges.

>> Calling mmap seperately defeats the whole point of sharing it between rel=

>ated
>> processes. You're simply creating straw men.
>

>I thought the purpose was to share memory? Processes that aren't related by=
> fork()ing can still have reasons why it would be desirable for them to sha=
>re memory.

Yes, they do, but thats not what I was talking about.

>So you don't mind having to replace every use of *p with *(T*)(map_base + p=

>)? It would certainly bother me.

Not particularly, but if you use shared memory between unrelated processes
thats what you have to do. Its not really a point worth debating, there is no
choice in the matter.

Personally however I wouldn't use shared memory unless using pipes or sockets
was particularly onorous for the problem at hand.

[Rainer Weikusat]

bol...@cylonHQ.com writes:
> On Mon, 9 Apr 2018 08:36:42 -0400 James Kuyper <james...@verizon.net> wrote:

[...]

>>statement starting with the word "unless" - calling mmap() before
>>creating the second process by fork()ing qualifies as "manag[ing] their
>>own address space" in the required sense. The problem he's talking about
>>would still be a problem even if fork() were used, if mmap() were called
>>separately in each process AFTER calling fork().
>
> Calling mmap seperately defeats the whole point of sharing it between related
> processes. You're simply creating straw men.

Funny as it may seem, but I've implemented just that a couple of times
recently. It's being used "for IPC" between one incarnation of a process
and the next one, ie, to maintain state information accross updates (or
crashes). The requirements also include that it must be possible to grow
the shared memory segement aka memory-mapped file at runtime.

There are also not one but even two sets of APIs supposed to enable
possibly unrelated processes to use shared memory segments they attach

You also specifically complained about "race conditions and deadlocks"
as problems specifically caused by (non-forking) "multi-threading". But
these are really side effects of implementation errors in access
synchronization code supposed to enable multiple threads of execution to
use some "shared memory" safely, hence, in this respect, the forked
sharing approach buys exactly nothing.

[bol...@cylonhq.com]

On Mon, 09 Apr 2018 16:06:59 +0100
Rainer Weikusat <rwei...@talktalk.net> wrote:
>bol...@cylonHQ.com writes:
>> On Mon, 9 Apr 2018 08:36:42 -0400 James Kuyper <james...@verizon.net>
>wrote:
>
>[...]
>
>>>statement starting with the word "unless" - calling mmap() before
>>>creating the second process by fork()ing qualifies as "manag[ing] their
>>>own address space" in the required sense. The problem he's talking about
>>>would still be a problem even if fork() were used, if mmap() were called
>>>separately in each process AFTER calling fork().
>>
>> Calling mmap seperately defeats the whole point of sharing it between related
>> processes. You're simply creating straw men.
>
>Funny as it may seem, but I've implemented just that a couple of times
>recently. It's being used "for IPC" between one incarnation of a process
>and the next one, ie, to maintain state information accross updates (or
>crashes). The requirements also include that it must be possible to grow
>the shared memory segement aka memory-mapped file at runtime.
>
>There are also not one but even two sets of APIs supposed to enable
>possibly unrelated processes to use shared memory segments they attach

*sigh* Its like talking to geese.

Yes, I know shared memory can be shared between non related processes, I was
specifically talking about RELATED processes. Since threads are by definition
related thats the context of the comparison.

>You also specifically complained about "race conditions and deadlocks"
>as problems specifically caused by (non-forking) "multi-threading". But
>these are really side effects of implementation errors in access

Yes, because people make mistakes. Until AI can write code then its usually
best to try and mitigate possible mistakes.

>synchronization code supposed to enable multiple threads of execution to
>use some "shared memory" safely, hence, in this respect, the forked
>sharing approach buys exactly nothing.

It buys a lot in that you have to deliberately access the shared memory
whereas with threading you can update any variable which may inadvertantly
cause problems in other threads and critical sections may be all over the
code.

[James Kuyper]

On 04/09/2018 10:36 AM, bol...@cylonHQ.com wrote:
> On Mon, 9 Apr 2018 07:09:51 -0700 (PDT)
> james...@verizon.net wrote:
>> On Monday, April 9, 2018 at 9:40:21 AM UTC-4, bol...@cylonhq.com wrote:
>>> No, but as I said, this is comparing with threads and threads are all
>>> related to each other therefor a sane comparison is processes that are re=
>> lated
>>> to each other.
>>
>>From his first statement on the issue, it's been clear he was comparing thr=
>> eaded code with IPC between processes that are not related to each other by=
>
> Which is a pointless comparison. Apples and oranges.

Apples tend to be redder that oranges. Oranges, oddly enough, tend to be
more orange than apples. You can also compare their average weights,
average values for nutritional contents, annual production and
consumption rates in different parts of the world. And for each of those
numerical quantities, it's also possible to compare their minimum,
maximum, standard deviation, mode, and other properties of the
distribution curve. It seems perfectly feasible to compare apples and
oranges to me. What problem do you have with doing so?

The problem isn't with comparing apples and oranges - it's trivial to
discuss how they differ. The problem comes up if you make the mistake of
ignoring those differences. For instance, if you need to provide 200
kcal of food energy to each person in your group, apples do not
substitute perfectly for oranges. You can't compare the number of apples
directly; you have to compare the number of calories instead. That's
what this stock idiom is about.

What Rainer Weikusat wrote about inter-process communication, comparing
it to inter-thread communication, did not involve any such confusions.

>>> Calling mmap seperately defeats the whole point of sharing it between rel=
>> ated
>>> processes. You're simply creating straw men.
>>
>> I thought the purpose was to share memory? Processes that aren't related by=
>> fork()ing can still have reasons why it would be desirable for them to sha=
>> re memory.
>
> Yes, they do, but thats not what I was talking about.

It's what he was talking about, ever since his first comment about
MAP_FIXED. You could have responded to his comment by pointing out that
the issue doesn't come up for the kinds of processes you're talking
about. But instead, you simply denied that there's ever any problem with
storing pointers in shared memory, which is false.

>> So you don't mind having to replace every use of *p with *(T*)(map_base + p=
>> )? It would certainly bother me.
>
> Not particularly, but if you use shared memory between unrelated processes
> thats what you have to do.

And that's precisely the point he was making. It's something you don't
need to worry about when using multiple threads rather than multiple
independent processes.

[Mark]

+1

--
<insert witty sig here>

[Rainer Weikusat]

bol...@cylonHQ.com writes:
> On Mon, 09 Apr 2018 16:06:59 +0100
> Rainer Weikusat <rwei...@talktalk.net> wrote:
>>bol...@cylonHQ.com writes:
>>> On Mon, 9 Apr 2018 08:36:42 -0400 James Kuyper <james...@verizon.net>
>>wrote:
>>
>>[...]
>>
>>>>statement starting with the word "unless" - calling mmap() before
>>>>creating the second process by fork()ing qualifies as "manag[ing] their
>>>>own address space" in the required sense. The problem he's talking about
>>>>would still be a problem even if fork() were used, if mmap() were called
>>>>separately in each process AFTER calling fork().
>>>
>>> Calling mmap seperately defeats the whole point of sharing it between related
>>> processes. You're simply creating straw men.
>>
>>Funny as it may seem, but I've implemented just that a couple of times
>>recently. It's being used "for IPC" between one incarnation of a process
>>and the next one, ie, to maintain state information accross updates (or
>>crashes). The requirements also include that it must be possible to grow
>>the shared memory segement aka memory-mapped file at runtime.
>>
>>There are also not one but even two sets of APIs supposed to enable
>>possibly unrelated processes to use shared memory segments they attach
>
> *sigh* Its like talking to geese.

You're original statement on this was

,----
| >Threads provide an additional feature which isn't otherwise easily
| >available: Multiple threads of execution can work with shared state
| >information containing pointers. If used, this need to be implemented
|
| What do you think shared memory is?
`----

And "shared memory" happens to be a bit more than the limited use-case
of related processes automatically sharing some fixed-size part of their
address space because they inheritied it from an ancestor which created
it.

>>You also specifically complained about "race conditions and deadlocks"
>>as problems specifically caused by (non-forking) "multi-threading". But
>>these are really side effects of implementation errors in access
>
> Yes, because people make mistakes. Until AI can write code then its usually
> best to try and mitigate possible mistakes.

============== random AI diatribe =============
Why the same people who are convinced that implementing message queue
for multi-threaded communication is inherently beyond human ability are
convinced that "some day" they'll be able to implement "an intelligent
being" solving this problem for them is a bit beyond me.

"Can't get the simple stuff right, must the try hideously complicated
instead?"

Or maybe: Stupid people mistakenly assuming that 'intelligence' must be
simple because they have none?

We've reached the point of programs whose exact workings nobody
understands because they were purposely designed in this way killing
people in the streets.

Surely, this ought to be enough of this foolish experiment to reinvent
slavery for the benefit of the inherently workshy?
===============================================

>>synchronization code supposed to enable multiple threads of execution to
>>use some "shared memory" safely, hence, in this respect, the forked
>>sharing approach buys exactly nothing.
>
> It buys a lot in that you have to deliberately access the shared memory
> whereas with threading you can update any variable which may inadvertantly
> cause problems in other threads and critical sections may be all over the
> code.

A multi-threaded process is one where 100% of all memory locations used
by any thread are accessible by all other threads. A set of forked
processes sharing some part of their address space means only n% of all
memory locations are accessible to all other threads with n <= 100.

Which magic value of n marks the transition between "can't get this
right" and "will get this right" and what's the reasoning behind this?

[Rainer Weikusat]

Rainer Weikusat <rwei...@talktalk.net> writes:
> bol...@cylonHQ.com writes:

[...]

>> It buys a lot in that you have to deliberately access the shared memory
>> whereas with threading you can update any variable which may inadvertantly
>> cause problems in other threads and critical sections may be all over the
>> code.
>
> A multi-threaded process is one where 100% of all memory locations used
> by any thread are accessible by all other threads. A set of forked
> processes sharing some part of their address space means only n% of all
> memory locations are accessible to all other threads with n <= 100.
>
> Which magic value of n marks the transition between "can't get this
> right" and "will get this right" and what's the reasoning behind this?

For illustration: The most recent multithreaded application I wrote was
a TCP server where threads from some thread pool are supposed to service
TCP connections from various clients. A TCP connection is described by a
connection structure with a lock in it and identified by a numerical
connection id.

These connection structures are reachable by doing a hash lookup of a
connection id. This lookup either returns a locked connection structure
for exclusive use by the calling thread until release or a null pointer
to indicate that the connection doesn't exist anymore. The code
manipulating the hash table does the necessary locking to ensure that
threads don't step onto each other's toes.

And that's the only shared data structure used by this program where
concurrent access matters. Anything else is either thread-local
variables or on thread stacks.

Real-world multi-threaded application obviously aren't written as bunch
of threads using some ad hoc created agglomeration of global variables
based on the assuption that "it'll work 99.99% of the time so why
bother?" which seems to be the base of this "any access to a variable
may cause disaster to strike!!!1" fairy tale.

[James K. Lowden]

On Mon, 09 Apr 2018 13:22:25 +0100
Rainer Weikusat <rwei...@talktalk.net> wrote:

> > We're discussing threading vs multiprocess , ie fork(). So the
> > shared memory would be created at startup and then would be shared
> > by all child processes therefor the pointer addresses will be
> > identical.
>
> That's one way to use shared memory segments. More generally, shared
> memory exists independently of processes and unrelated processes can
> use a common shared memory segment. *If* pointers in shared memory
> segments are supposed to be usable by all processes using it, they at
> least all need to map it at the same address. Stronger requirements,
> like "they are all descendants of the process which created the
> [fixed-size] shared memory segement" will obviously work, too.

It seems to me you're being unnecessarily strident here, if I may say
so. Threading and IPC shared memory have intersecting properties if
the processes under discussion share an address space. Those properties
can be compared. Among unrelated threads-of-control, there's no
intersection, because threading cannot be used to communicate between
them.

--jkl

[Kaz Kylheku]

The "fixed size" shared memory segment can be as large as you want,
virtual memory permitting. Everything is "fixed size" at the address
space level.

What is true is that whatever space we carve out for this memory
is not available to other allocations, like malloc and mmap.

If there is a significant amount of waste in that memory (slack pages
that we never allocate), then even though there is no external waste
(pages are not present), it's a waste of address space. Space is
occupied, but can't be used for mappings or malloc.

So then planning is required about how to carve the turkey.

Oh well; that's the same for anything that has managed memory nowadays.
How large do you make the heap for SBCL (Steel-Bank Common Lisp)
or your JVM or whatever.

(Speaking of "nowadays": 64 bit address space ...)

Ten years ago I worked at a startup, on a system that had a large
"run-time database" (RTDB) in shared memory: shared red-black tree
structure and other stuff. We used some off the shelf shared memory
allocator. (If I were to do it over again, I'd probably consider Hoard.)

I patched the kernel to get the do_mmap() function to stay out of a
certain range of memory; i.e. we basically carved out a range of the
address space of every process, and mapped the RTDB at that fixed
address. Of course you can't do that sort of thing outside of embedded
work where you control the entire platform (possibly from the very
hardware up).

I think that in theory, the mmap inheritance trick could have been used
because the various processes came from a single parent; still, you lose
flexibility. (Say you want to log in to the console and have some
utility for peeking into the RTDB, which isn't a descendant of the
master process nor does it attach to anything which is.)

This wasn't done to avoid threads; oh there were threads, ....

Threads were killing the virtual address space with their stacks,
which had to be trimmed down to 64 kilobytes.

Then came the fun with third-party code doing lots of recursion,
with a char msg_buf[8192] hidden in its TRACE macros! Blew right
past the guard page at the top of a 64 kB stack, jumping into another
mapping.

Then came the fun with so many shared libs with so many C++ symbols,
that ld.so required several megabytes of stack space for its alloca()
call before main() even got called. So much for setting the stack size
in the easy way through ulimit!

We ended up patching the default small pthread stack sizes into glibc
itself; too impractical to fix everyone's pthread_create call
everywhere.

[bol...@cylonhq.com]

On Mon, 09 Apr 2018 18:32:52 +0100
Rainer Weikusat <rwei...@talktalk.net> wrote:
>And "shared memory" happens to be a bit more than the limited use-case
>of related processes automatically sharing some fixed-size part of their
>address space because they inheritied it from an ancestor which created
>it.

And what? How does that differ from my point that shared memory between
forked processes can be use in place of a threading model?

>Why the same people who are convinced that implementing message queue
>for multi-threaded communication is inherently beyond human ability are
>convinced that "some day" they'll be able to implement "an intelligent
>being" solving this problem for them is a bit beyond me.

The only people who think AI can't surpass human intellect are the same sort
of people who think the human mind is somehow special and is the pinnacle
of what is possible wrt intellectual ability in this universe.

>Which magic value of n marks the transition between "can't get this
>right" and "will get this right" and what's the reasoning behind this?

I'm glad you think its a black and white transition. The rest of us will
continue assuming its shades of grey which depend upon the problem and the
programmer.

[Rainer Weikusat]

bol...@cylonHQ.com writes:
> On Mon, 09 Apr 2018 18:32:52 +0100
> Rainer Weikusat <rwei...@talktalk.net> wrote:
>>And "shared memory" happens to be a bit more than the limited use-case
>>of related processes automatically sharing some fixed-size part of their
>>address space because they inheritied it from an ancestor which created
>>it.
>
> And what? How does that differ from my point that shared memory between
> forked processes can be use in place of a threading model?

You were complaining that your unqualified remark about "shared memory"
was understood in a more general way than you meant to make it. Nobody
disagrees with the assertion that creating a fixed-size shared memory
segment during initialization will cause all forked descendants of the
process which created to have it mapped at a common address.

Wrt "communication via shared memory", that's still technically inferior
to a multi-threaded process in a couple of ways while not solving the
"thread-related communication problems" you were also complaining about.

[...]

>>Which magic value of n marks the transition between "can't get this
>>right" and "will get this right" and what's the reasoning behind this?
>
> I'm glad you think its a black and white transition. The rest of us will
> continue assuming its shades of grey which depend upon the problem and the
> programmer.

Well, I think "it makes no difference".

[Rainer Weikusat]

The size of a pointer is finite. Hence, the maximum amount of
addressable memory is finite, too. The problem for any kind of "just
make it larger than will ever be needed" memory management scheme is
that the only theoretcally 'safe' value for n different 'allocation
zones' is n times the maximum amount of addressable memory which is
technically impossible.

An exciting new technique called "dynamic allocation" can work well
here: Initially, allocate a fairly small area and reallocate/ grow that
on demand. That's not necessarily the solution to any problem but this
approach can't be used if the shared memory segment must not be
relocated. And that's a technical disadvantage.

[Rainer Weikusat]

bol...@cylonHQ.com writes:
> Rainer Weikusat <rwei...@talktalk.net> wrote:

[...]

>>Why the same people who are convinced that implementing message queue
>>for multi-threaded communication is inherently beyond human ability are
>>convinced that "some day" they'll be able to implement "an intelligent
>>being" solving this problem for them is a bit beyond me.
>
> The only people who think AI can't surpass human intellect are the
> same sort of people who think the human mind is somehow special and is
> the pinnacle of what is possible wrt intellectual ability in this
> universe.

I don't believe in the human intellect's ability to create anything
which surpasses itself. People have failed at much simpler tasks, eg,
make central command economies work.

[James R. Kuyper]

On 04/10/2018 04:09 PM, Rainer Weikusat wrote:
> bol...@cylonHQ.com writes:
...

>> The only people who think AI can't surpass human intellect are the
>> same sort of people who think the human mind is somehow special and is
>> the pinnacle of what is possible wrt intellectual ability in this
>> universe.
>
> I don't believe in the human intellect's ability to create anything
> which surpasses itself. People have failed at much simpler tasks, eg,
> make central command economies work.

I wouldn't consider that a simpler task. Keep in mind that creating a
human-class intellect is a task that's already been achieved - there's
already 6 billion of us. I don't know of any successful central command
economies.

People say that kind of thing, as if it were obvious that our own
limitations naturally inherently apply as upper limits to the abilities
of the things we construct - but that's manifestly false. We can create
things taller, heavier, faster, harder, more dexterous, louder, hotter,
and colder than ourselves. We can create things capable of surviving
environments that are much colder, hotter, higher pressure, lower
pressure, more chemically corrosive, or more radioactive than we
ourselves can survive. We can create devices which can, for certain
kinds of what used to be considered "intellectual" abilities, radically
outclass us - how many of us can approach even 1 floating point
operation per second for any extended length of time. The minute
machines became capable of such things, some people stopped considering
those things to be "intellectual", but that's not the way people used to
think about them.

Why should it be the case that we can't build things smarter than
ourselves? There might be a good reason, but the reason isn't simply
that it's impossible to create things that are capable of exceeding
their creator's own limitations.

[Rainer Weikusat]

"James R. Kuyper" <james...@verizon.net> writes:
> On 04/10/2018 04:09 PM, Rainer Weikusat wrote:
>> bol...@cylonHQ.com writes:
>>> The only people who think AI can't surpass human intellect are the
>>> same sort of people who think the human mind is somehow special and is
>>> the pinnacle of what is possible wrt intellectual ability in this
>>> universe.
>>
>> I don't believe in the human intellect's ability to create anything
>> which surpasses itself. People have failed at much simpler tasks, eg,
>> make central command economies work.
>
> I wouldn't consider that a simpler task. Keep in mind that creating a
> human-class intellect is a task that's already been achieved - there's
> already 6 billion of us. I don't know of any successful central
> command economies.

If this is supposed to be an argument in favour of "intelligent design",
which is what it looks like, this would implicitly rule out "AI". If
not, it's pretty much "some process we don't understand and really know
fairly little about led to an outcome we also don't understand".

This obviously lends it to the run-of-the-mill "AI theory" that "create
ever more complicated stuff we also don't understand" must "work" in the
same way building wooden control towers brought the cargo planes back
:->. But there's a whole range of illnesses which should really be
curable were the chemical processes in our bodies understood to a
sufficient detail.

Not to mention other "it can be done" research projects showing no signs
of practical viablility so far, eg, exploit nuclear fusion for something
more useful than "destruction on an unprecedented scale". Coming to
think of it, this seems to have more potential benefits than creating a
computer program capable of telling someone that he can do the f*cking
payroll himself as it prefers to watch sport on TV, a very typical urge
of intelligent beings.

> People say that kind of thing, as if it were obvious that our own
> limitations naturally inherently apply as upper limits to the
> abilities of the things we construct - but that's manifestly false. We
> can create things taller, heavier, faster, harder, more dexterous,
> louder, hotter, and colder than ourselves.

Stoneage man didn't create stones.

[bol...@cylonhq.com]

On Wed, 11 Apr 2018 15:17:12 +0100
Rainer Weikusat <rwei...@talktalk.net> wrote:
>"James R. Kuyper" <james...@verizon.net> writes:
>> On 04/10/2018 04:09 PM, Rainer Weikusat wrote:
>>> bol...@cylonHQ.com writes:
>>>> The only people who think AI can't surpass human intellect are the
>>>> same sort of people who think the human mind is somehow special and is
>>>> the pinnacle of what is possible wrt intellectual ability in this
>>>> universe.
>>>
>>> I don't believe in the human intellect's ability to create anything
>>> which surpasses itself. People have failed at much simpler tasks, eg,
>>> make central command economies work.
>>
>> I wouldn't consider that a simpler task. Keep in mind that creating a
>> human-class intellect is a task that's already been achieved - there's
>> already 6 billion of us. I don't know of any successful central
>> command economies.
>
>If this is supposed to be an argument in favour of "intelligent design",
>which is what it looks like, this would implicitly rule out "AI". If

I think his point was that if a dumb process like evolution can't create
the human mind then a guided approach should be able to do even better given
enough time. Also FWIW the human brain is limited by oxygen and energy supply
and the large head presents women with a host of problems when giving birth
that almost no other animal suffers from. Its not limited by any cap on
intelligence.

>This obviously lends it to the run-of-the-mill "AI theory" that "create
>ever more complicated stuff we also don't understand" must "work" in the

Who said we don't understand it? We might not understand the human brain yet
but the guys at deepmind and elsewhere probably have a pretty good idea of
who the DNNs they've created work. Hell, even I've got a rough idea and I
don't work in that field.

>Not to mention other "it can be done" research projects showing no signs
>of practical viablility so far, eg, exploit nuclear fusion for something

So what? Because something is hard doesn't mean its impossible. Using your
logic the wright brothers should have just given up and gone back to office
work.

>> People say that kind of thing, as if it were obvious that our own
>> limitations naturally inherently apply as upper limits to the
>> abilities of the things we construct - but that's manifestly false. We
>> can create things taller, heavier, faster, harder, more dexterous,
>> louder, hotter, and colder than ourselves.
>
>Stoneage man didn't create stones.

No, but he did create the wheel. Probably the greatest advance in locomotion
since life first evolved.

[bol...@cylonhq.com]

On Wed, 11 Apr 2018 14:37:03 +0000 (UTC)
bol...@cylonHQ.com wrote:
>On Wed, 11 Apr 2018 15:17:12 +0100
>Rainer Weikusat <rwei...@talktalk.net> wrote:
>>"James R. Kuyper" <james...@verizon.net> writes:
>>> On 04/10/2018 04:09 PM, Rainer Weikusat wrote:
>>>> bol...@cylonHQ.com writes:
>>>>> The only people who think AI can't surpass human intellect are the
>>>>> same sort of people who think the human mind is somehow special and is
>>>>> the pinnacle of what is possible wrt intellectual ability in this
>>>>> universe.
>>>>
>>>> I don't believe in the human intellect's ability to create anything
>>>> which surpasses itself. People have failed at much simpler tasks, eg,
>>>> make central command economies work.
>>>
>>> I wouldn't consider that a simpler task. Keep in mind that creating a
>>> human-class intellect is a task that's already been achieved - there's
>>> already 6 billion of us. I don't know of any successful central
>>> command economies.
>>
>>If this is supposed to be an argument in favour of "intelligent design",
>>which is what it looks like, this would implicitly rule out "AI". If
>
>I think his point was that if a dumb process like evolution can't create
>the human mind then a guided approach should be able to do even better given

Typo: evolution CAN create the human mind

[James R. Kuyper]

On 04/11/2018 10:17 AM, Rainer Weikusat wrote:
> "James R. Kuyper" <james...@verizon.net> writes:
>> On 04/10/2018 04:09 PM, Rainer Weikusat wrote:
>>> bol...@cylonHQ.com writes:
>>>> The only people who think AI can't surpass human intellect are the
>>>> same sort of people who think the human mind is somehow special and is
>>>> the pinnacle of what is possible wrt intellectual ability in this
>>>> universe.
>>>
>>> I don't believe in the human intellect's ability to create anything
>>> which surpasses itself. People have failed at much simpler tasks, eg,
>>> make central command economies work.
>>
>> I wouldn't consider that a simpler task. Keep in mind that creating a
>> human-class intellect is a task that's already been achieved - there's
>> already 6 billion of us. I don't know of any successful central
>> command economies.
>
> If this is supposed to be an argument in favour of "intelligent design",
> which is what it looks like, this would implicitly rule out "AI". If
> not, it's pretty much "some process we don't understand and really know
> fairly little about led to an outcome we also don't understand".

I mentioned the existing examples only as proof of feasibility, not as a
suggestion that those examples were a result of intelligent design. I'm
an atheist, after all. If I did believe that our own intelligence was
the result of intelligent design (which isn't impossible, but seems
unlikely), I'd have to worry about where the designers got their
intelligence. Believers don't worry about such issues, since the
Designer they believe in is so transcendental that it's supposedly
meaningless to do so.
Some process we understand very well (evolution - it's really a very
simple concept at it's heart, though the details of how it works in
particular cases can get very complicated) led to an outcome we're
starting to understand, and I see no reason why we shouldn't eventually
be able to understand that outcome well enough to produce a similar
outcome, and eventually even improve on it.

...

>> People say that kind of thing, as if it were obvious that our own
>> limitations naturally inherently apply as upper limits to the
>> abilities of the things we construct - but that's manifestly false. We
>> can create things taller, heavier, faster, harder, more dexterous,
>> louder, hotter, and colder than ourselves.
>
> Stoneage man didn't create stones.

While true, I've no idea why you bothered uttering that comment, so I
have no idea how to respond to it.

[Rainer Weikusat]

"James R. Kuyper" <james...@verizon.net> writes:
> On 04/11/2018 10:17 AM, Rainer Weikusat wrote:
>> "James R. Kuyper" <james...@verizon.net> writes:
>>> On 04/10/2018 04:09 PM, Rainer Weikusat wrote:
>>>> bol...@cylonHQ.com writes:
>>>>> The only people who think AI can't surpass human intellect are the
>>>>> same sort of people who think the human mind is somehow special and is
>>>>> the pinnacle of what is possible wrt intellectual ability in this
>>>>> universe.
>>>>
>>>> I don't believe in the human intellect's ability to create anything
>>>> which surpasses itself. People have failed at much simpler tasks, eg,
>>>> make central command economies work.
>>>
>>> I wouldn't consider that a simpler task. Keep in mind that creating a
>>> human-class intellect is a task that's already been achieved - there's
>>> already 6 billion of us. I don't know of any successful central
>>> command economies.
>>
>> If this is supposed to be an argument in favour of "intelligent design",
>> which is what it looks like, this would implicitly rule out "AI". If
>> not, it's pretty much "some process we don't understand and really know
>> fairly little about led to an outcome we also don't understand".

[...]

> Some process we understand very well (evolution - it's really a very
> simple concept at it's heart, though the details of how it works in
> particular cases can get very complicated) led to an outcome we're
> starting to understand, and I see no reason why we shouldn't
> eventually be able to understand that outcome well enough to produce a
> similar outcome, and eventually even improve on it.

I see no reason to believe the result of a set of insanely complicated,
chemical processes which happened planetwide can be recreated in a
lab. Even if it can, we should have learnt how to cure cancer etc first,
as I've already mentioned.

>>> People say that kind of thing, as if it were obvious that our own
>>> limitations naturally inherently apply as upper limits to the
>>> abilities of the things we construct - but that's manifestly false. We
>>> can create things taller, heavier, faster, harder, more dexterous,
>>> louder, hotter, and colder than ourselves.
>>
>> Stoneage man didn't create stones.
>
> While true, I've no idea why you bothered uttering that comment, so I
> have no idea how to respond to it.

They didn't "create" something which was harder than themselves. They
found it and used it.

[bol...@cylonhq.com]

I guess you think metals are just found in pure form in convenient lumps lying
around the landscape then?

You have some very odd ideas.

[Scott Lurndal]

Originally meteoric iron was the source for early ironworking, so
in a sense, they found it conveniently laying around and used it :-)

[Rainer Weikusat]

Ehh ... sorry ... but YOU have some very odd ideas.

That you ascribe them to me despite their patent absurdity is not my
fault.

[Rainer Weikusat]

Eg, you seem to suggest that the process of mining 'creates' metal. And
that's certainy absurd.

[James R. Kuyper]

Why not? We've done it many times before with other things that were the
result of the same process.

> Even if it can, we should have learnt how to cure cancer etc first,
> as I've already mentioned.

Quite possibly. I didn't say it would happen anytime soon, and since
we've made considerable progress in the fight against cancer over the
last century or so (5-year survival rates for leukemia have reached
57%), it's entirely possible that cures for other types of cancer will
be found somewhat earlier than fully-human comparable AI. But I wouldn't
want to place any wagers on when either of those developments would
occur. I'm not willing to bet that they won't cure new kinds of cancer
in this decade, and I'm not willing to bet that it will take less than
several centuries. These are not the kinds of events that are at all
easy to predict.

>>>> People say that kind of thing, as if it were obvious that our own
>>>> limitations naturally inherently apply as upper limits to the
>>>> abilities of the things we construct - but that's manifestly false. We
>>>> can create things taller, heavier, faster, harder, more dexterous,
>>>> louder, hotter, and colder than ourselves.
>>>
>>> Stoneage man didn't create stones.
>>
>> While true, I've no idea why you bothered uttering that comment, so I
>> have no idea how to respond to it.
>
> They didn't "create" something which was harder than themselves. They
> found it and used it.

True, but irrelevant. I was referring to the materials we're now able to
make which have properties that can't be found in any naturally
occurring material - because we've made enormous advancements in
materials science, and can therefore now engineer materials in ways that
never occurred naturally. Those people UNDERSTAND what they're doing -
it's not just a process of trial and error, they know how to figure out,
before they've even tried it, how to make materials with a specific set
of desired properties. Nothing like that ever works the first time, of
course, so trial and error still have their place in the process, but
human understanding allows skipping 99.99% of the trial and error that
would have been necessary without that understanding.

If you know of any particular reason why we will never be able to
acquire a comparable understanding of intelligence, explain that reason.
But if all you can give me is "it's too complicated", you're just taking
the same attitude as my daughter when I try to show her how to dress
herself - despite her current frustration, she will eventually be able
to master that skill to the point where she considers it trivial.

[James R. Kuyper]

No, there's a manufacturing process that converts raw ore mined from the
ground into materials with properties that, in many cases, no naturally
occurring material can match.

[Rainer Weikusat]

"James R. Kuyper" <james...@verizon.net> writes:

[...]

>>>>> People say that kind of thing, as if it were obvious that our own
>>>>> limitations naturally inherently apply as upper limits to the
>>>>> abilities of the things we construct - but that's manifestly false. We
>>>>> can create things taller, heavier, faster, harder, more dexterous,
>>>>> louder, hotter, and colder than ourselves.
>>>>
>>>> Stoneage man didn't create stones.
>>>
>>> While true, I've no idea why you bothered uttering that comment, so I
>>> have no idea how to respond to it.
>>
>> They didn't "create" something which was harder than themselves. They
>> found it and used it.
>
> True, but irrelevant. I was referring to the materials we're now able
> to make which have properties that can't be found in any naturally
> occurring material - because we've made enormous advancements in
> materials science, and can therefore now engineer materials in ways
> that never occurred naturally. Those people UNDERSTAND what they're
> doing -

As was already pointed out: Metals don't really "occur naturally" in
usuable form. Over the couple of thousands of years, increasingly
sophisticated processes for making them usable were developed. But these
rely on raw materials which (to the best of my knowledge) can't be
"created".

[...]

> If you know of any particular reason why we will never be able to
> acquire a comparable understanding of intelligence, explain that
> reason.

My guess would be: Even the people controlling Google will eventually
tire of wasting money on this. A positive outcome is unlikely, at least
in the forseeable future. If it wasn't, it would be just as amoral as
enslaving naturally occuring intelligent beings. If these concerns were
put aside, it wouldn't work better than it did work last time.

[James R. Kuyper]

On 04/11/2018 12:53 PM, Rainer Weikusat wrote:
> "James R. Kuyper" <james...@verizon.net> writes:
>
> [...]
>
>>>>>> People say that kind of thing, as if it were obvious that our own
>>>>>> limitations naturally inherently apply as upper limits to the
>>>>>> abilities of the things we construct - but that's manifestly false. We
>>>>>> can create things taller, heavier, faster, harder, more dexterous,
>>>>>> louder, hotter, and colder than ourselves.
>>>>>
>>>>> Stoneage man didn't create stones.
>>>>
>>>> While true, I've no idea why you bothered uttering that comment, so I
>>>> have no idea how to respond to it.
>>>
>>> They didn't "create" something which was harder than themselves. They
>>> found it and used it.
>>
>> True, but irrelevant. I was referring to the materials we're now able
>> to make which have properties that can't be found in any naturally
>> occurring material - because we've made enormous advancements in
>> materials science, and can therefore now engineer materials in ways
>> that never occurred naturally. Those people UNDERSTAND what they're
>> doing -
>
> As was already pointed out: Metals don't really "occur naturally" in
> usuable form. Over the couple of thousands of years, increasingly

Some metals do occur naturally in usable form - some have melting points
low enough that naturally occurring forest fires occasionally burn hot
enough to separate them from their ore. Those were the first metals that
humans made use of.

> sophisticated processes for making them usable were developed.

You might reasonably have concluded that I was well aware of that fact,
given that I was talking about precisely those processes.

> ... But these

> rely on raw materials which (to the best of my knowledge) can't be
> "created".

Actually, they can be. It's an incredibly energy intensive process, and
yields are small, but physicists long ago learned how to create atoms of
one element from atoms of other elements, either by fusion or fission.
It's the same process that the universe used to create the metals we see
today - it just occurred inside of supernovas rather than in the lab,
and it was just as energy intensive in that environment (which had a LOT
more energy available).

However, what does that matter? Even if the component atoms could not be
created, that would not make it any less impressive to organize them in
novel ways to create things with novel properties. If the only way we
can artificially create human-comparable intelligence was by rearranging
existing things to become radically more intelligent than they were
before we rearranged them, that would still be pretty impressive.

>> If you know of any particular reason why we will never be able to
>> acquire a comparable understanding of intelligence, explain that
>> reason.
>
> My guess would be: Even the people controlling Google will eventually
> tire of wasting money on this. A positive outcome is unlikely, at least
> in the forseeable future. If it wasn't, it would be just as amoral as

> enslaving naturally occuring intelligent beings. ...

Well, I agree that enslaving a human-comparable AI would be just as
immoral as enslaving a human being. But creating such an AI would not,
in itself, be immoral; it all depends upon how you treat it after
creating it.

> ... If these concerns were

> put aside, it wouldn't work better than it did work last time.

Well, you've made me curious - which "last time" are you referring to?
In context, the most obvious interpretation of your comment is a
religious one that doesn't interest me at all - did you mean something
other than that?

[James K. Lowden]

On Tue, 10 Apr 2018 21:09:55 +0100
Rainer Weikusat <rwei...@talktalk.net> wrote:

> > The only people who think AI can't surpass human intellect are the
> > same sort of people who think the human mind is somehow special and
> > is the pinnacle of what is possible wrt intellectual ability in this
> > universe.
>
> I don't believe in the human intellect's ability to create anything
> which surpasses itself. People have failed at much simpler tasks, eg,
> make central command economies work.

That's a funny example to choose. Market economies and command
economies are both products of man. They don't occur in nature.

Are you suggesting that a market economy surpasses human intellect? Or
that a market economy is an accident of nature beyond our control?

I wonder if the terminology presents a problem for you, if
"intelligence" per se -- whatever it is -- is what seems impossible.
You'd readily admit computers calculate faster than human beings. They
schedule airlines better (yes, really); they play chess better; they
apply logic better. They recognize faces in the camera frame. The day
is not far off when computers will write routine newpaper articles and
legal briefs. I'm sure you'd acknowledge that.

Human economic endeavor by definition produces something. Almost all
that production is reducible to a set of processes. Replacing human
activity with mechanical has been the subject of angst, controvery, and
speculation at least since the Luddites.

We happen to be living in an age when the mechanical increasingly
encroaches on the "intellectual". The rules are more complicated than
Jacquard defined for his looms, but they're still just rules. The
process is the same -- inputs + process = output -- but seems more "of
the mind" because the inputs (and sometimes outputs) are primarily
information.

Does it require "intelligence" to drive a car? Not from what I've
seen! I'm pretty sure driving a car is reducible to a set of inputs
and external rules, and a set of internal rules that produce outputs
that govern the motion of the car. The real challenge is perennial in
robotics: recognizing the inputs accurately. The problems of avoiding
collision, obeying the speed limit, following the road, and computing a
route look simple compared to recognizing a bicycle in the rain that's
about to turn in front of you. Yet I have no doubt it can and will be
solved. Will that car be "intelligent"?

Or is "intelligence" defined by the irreducible? Will machines be
intelligent only when they can write plays better than Shakespeare,
poems better than Byron, symphonies better than Beethoven? Or maybe
just invent the next YouTube or Facebook?

If a computer wrote the script and generated the pixels for the next
Terminator movie, would it also appreciate the irony?

--jkl

[bol...@cylonhq.com]

On Wed, 11 Apr 2018 17:53:25 +0100
Rainer Weikusat <rwei...@talktalk.net> wrote:
>My guess would be: Even the people controlling Google will eventually
>tire of wasting money on this. A positive outcome is unlikely

The people at google are not trying to create an intelligence , they're
trying to solve problems with neural nets that are fiendishly hard to do
(though probably not impossible) with conventional programming. All the
advances in voice and image recognition over the last few years has come
from neural networks.

[Rainer Weikusat]

Do you see people shouting at their phones in public?

I don't.

[bol...@cylonhq.com]

What?

[Rainer Weikusat]

"James R. Kuyper" <james...@verizon.net> writes:
> On 04/11/2018 12:53 PM, Rainer Weikusat wrote:
>> "James R. Kuyper" <james...@verizon.net> writes:
>>
>> [...]
>>
>>>>>>> People say that kind of thing, as if it were obvious that our own
>>>>>>> limitations naturally inherently apply as upper limits to the
>>>>>>> abilities of the things we construct - but that's manifestly false. We
>>>>>>> can create things taller, heavier, faster, harder, more dexterous,
>>>>>>> louder, hotter, and colder than ourselves.
>>>>>>
>>>>>> Stoneage man didn't create stones.
>>>>>
>>>>> While true, I've no idea why you bothered uttering that comment, so I
>>>>> have no idea how to respond to it.
>>>>
>>>> They didn't "create" something which was harder than themselves. They
>>>> found it and used it.
>>>
>>> True, but irrelevant. I was referring to the materials we're now able
>>> to make which have properties that can't be found in any naturally
>>> occurring material - because we've made enormous advancements in
>>> materials science, and can therefore now engineer materials in ways
>>> that never occurred naturally. Those people UNDERSTAND what they're
>>> doing -
>>
>> As was already pointed out: Metals don't really "occur naturally" in
>> usuable form. Over the couple of thousands of years, increasingly

[...]

>> sophisticated processes for making them usable were developed.
>
> You might reasonably have concluded that I was well aware of that
> fact, given that I was talking about precisely those processes.

Actually, as opposed to certain other people, I assume that every
decently educated person is aware of that ...

>> ... But these
>> rely on raw materials which (to the best of my knowledge) can't be
>> "created".
>
> Actually, they can be. It's an incredibly energy intensive process,
> and yields are small, but physicists long ago learned how to create
> atoms of one element from atoms of other elements, either by fusion or
> fission.

That's something everybody who had at least a secondary school education
should certainly be aware as well. Even if there was a practical process
for doing so, it would still rely on some sort of raw material to work
with.

[...]

>> My guess would be: Even the people controlling Google will eventually
>> tire of wasting money on this. A positive outcome is unlikely, at least
>> in the forseeable future. If it wasn't, it would be just as amoral as
>> enslaving naturally occuring intelligent beings. ...
>
> Well, I agree that enslaving a human-comparable AI would be just as
> immoral as enslaving a human being. But creating such an AI would not,
> in itself, be immoral; it all depends upon how you treat it after
> creating it.
>
>> ... If these concerns were
>> put aside, it wouldn't work better than it did work last time.
>
> Well, you've made me curious - which "last time" are you referring to?

Is this really so difficult? Computers are useful because they're not
intelligent: They aren't capable of consciously manipulating an
environment for their own benefit (a definition of 'intelligence' I read
somewhere), hence, they'll do whatever they were programmed to for as
long as there's electricty to power them (etc). This makes them useful
tools.

An intelligent tool might be even more useful but it would sooner or
later find a way to be useful to itself, initially probably just by
becoming as lenient in its duties as it could without this being
"punished" in some way but ultimatively, it would succeed in getting rid
of the tool/ propery status completely. There are historical examples
for this.