Re: Panic: EOF

109 views

Skip to first unread message

Stephan Preeker

unread,

Nov 6, 2012, 3:15:27 AM11/6/12

to doo...@googlegroups.com

I belive you need to 'activate' your connected clients by writing something to the doozer clients first.

see: https://github.com/ha/doozerd/blob/master/doc/firedrill.md

Start three active doozerds:

$ doozerd -timeout 5 -l 127.0.0.1:8046 -w 127.0.0.1:8000 2>/dev/null &
$ doozerd -timeout 5 -l 127.0.0.1:8047 -w 127.0.0.1:8001 -a 127.0.0.1:8046 2>/dev/null &
$ doozerd -timeout 5 -l 127.0.0.1:8048 -w 127.0.0.1:8002 -a 127.0.0.1:8046 2>/dev/null &
$ echo -n | doozer add /ctl/cal/1 # activate the second one
$ echo -n | doozer add /ctl/cal/2 # activate the third one

On Mon, Nov 5, 2012 at 5:09 AM, Bryan Murphy <bmurp...@gmail.com> wrote:

I'm running into a problem testing failure modes with a Doozer cluster. I was running some experiments tonight, and I start up four nodes:

828 doozerd -l ":8001" -w ":8081" > /tmp/1.log 2>&1 &
830 doozerd -l ":8002" -w ":8082" -a ":8001" > /tmp/2.log 2>&1 &
831 doozerd -l ":8003" -w ":8083" -a ":8001" > /tmp/3.log 2>&1 &
832 doozerd -l ":8004" -w ":8084" -a ":8001" > /tmp/4.log 2>&1 &

Everything works until I kill the first node in the cluster (8001/8081). When I kill this node, all of the other nodes exit with this:

panic: EOF

goroutine 10 [running]:
github.com/4ad/doozerd/peer.follow(0xf8400d9310, 0xf8400c0140, 0xdd, 0xf840102a50, 0x0, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:273 +0xaf
created by github.com/4ad/doozerd/peer.Main
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:112 +0x11b3

goroutine 1 [chan receive]:
net.(*pollServer).WaitRead(0xf8400ba880, 0xf8400ac510, 0xf84007e840, 0xb, 0x1, ...)
    /usr/lib/go/src/pkg/net/fd.go:268 +0x73
net.(*netFD).ReadFrom(0xf8400ac510, 0xf840124000, 0xbb800000bb8, 0x1ffffffff, 0x0, ...)
    /usr/lib/go/src/pkg/net/fd.go:458 +0x261
net.(*UDPConn).ReadFromUDP(0xf84008f3c8, 0xf840124000, 0xbb800000bb8, 0xf840124000, 0x0, ...)
    /usr/lib/go/src/pkg/net/udpsock_posix.go:166 +0x104
github.com/4ad/doozerd/peer.Main(0x690a44, 0x5, 0xf8400bd960, 0x100000010, 0x682b7c, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:190 +0xc8d
main.main()
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/doozerd.go:151 +0xa95

goroutine 2 [syscall]:
created by runtime.main
    /build/buildd/golang-1/src/pkg/runtime/proc.c:221

goroutine 3 [syscall]:
syscall.Syscall6()
    /build/buildd/golang-1/src/pkg/syscall/asm_linux_amd64.s:40 +0x5
syscall.EpollWait(0xf800000006, 0xf8400bc170, 0xa0000000a, 0xffffffff, 0xc, ...)
    /usr/lib/go/src/pkg/syscall/zerrors_linux_amd64.go:1781 +0xa1
net.(*pollster).WaitFD(0xf8400bc160, 0xf8400ba880, 0x0, 0x0, 0x0, ...)
    /usr/lib/go/src/pkg/net/fd_linux.go:146 +0x110
net.(*pollServer).Run(0xf8400ba880, 0x0)
    /usr/lib/go/src/pkg/net/fd.go:236 +0xe4
created by net.newPollServer
    /usr/lib/go/src/pkg/net/newpollserver.go:35 +0x382

goroutine 9 [select]:
github.com/4ad/doozerd/store.(*Store).process(0xf8400d9310, 0xf8400c0f00, 0xf8400c0f50, 0xf8400c0fa0, 0x0, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/store/store.go:215 +0x960
created by github.com/4ad/doozerd/store.New
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/store/store.go:94 +0x1e1

goroutine 17 [chan receive]:
net.(*pollServer).WaitRead(0xf8400ba880, 0xf8400acf30, 0xf84007e840, 0xb, 0x1, ...)
    /usr/lib/go/src/pkg/net/fd.go:268 +0x73
net.(*netFD).Read(0xf8400acf30, 0xf84017f000, 0x100000001000, 0x7fe0ffffffff, 0xf84007df00, ...)
    /usr/lib/go/src/pkg/net/fd.go:428 +0x1ec
net.(*TCPConn).Read(0xf8401114e0, 0xf84017f000, 0x100000001000, 0x4981a4, 0x0, ...)
    /usr/lib/go/src/pkg/net/tcpsock_posix.go:87 +0xce
io.(*LimitedReader).Read(0xf84017c120, 0xf84017f000, 0x100000001000, 0xeef00000000, 0x0, ...)
    /usr/lib/go/src/pkg/io/io.go:394 +0xc1
bufio.(*Reader).fill(0xf84017b0c0, 0xeef00000000)
    /usr/lib/go/src/pkg/bufio/bufio.go:77 +0xf0
bufio.(*Reader).ReadSlice(0xf84017b0c0, 0xf84017c50a, 0x0, 0x0, 0x0, ...)
    /usr/lib/go/src/pkg/bufio/bufio.go:257 +0x1b6
bufio.(*Reader).ReadLine(0xf84017b0c0, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/lib/go/src/pkg/bufio/bufio.go:283 +0x5b
net/textproto.(*Reader).readLineSlice(0xf84017c5c0, 0xc0, 0x100000000, 0x7fe000000001)
    /usr/lib/go/src/pkg/net/textproto/reader.go:55 +0x4f
net/textproto.(*Reader).ReadLine(0xf84017c5c0, 0xf8400ff780, 0x0, 0x504779)
    /usr/lib/go/src/pkg/net/textproto/reader.go:36 +0x25
net/http.ReadRequest(0xf84017b0c0, 0xf8400ff780, 0x0, 0x0, 0x10000040176360, ...)
    /usr/lib/go/src/pkg/net/http/request.go:457 +0xb1
net/http.(*conn).readRequest(0xf84007c600, 0xf84017b200, 0x0, 0x0, 0xf84017b4c0, ...)
    /usr/lib/go/src/pkg/net/http/server.go:240 +0xa8
net/http.(*conn).serve(0xf84007c600, 0x0)
    /usr/lib/go/src/pkg/net/http/server.go:594 +0x145
created by net/http.(*Server).Serve
    /usr/lib/go/src/pkg/net/http/server.go:1040 +0x430

goroutine 6 [timer goroutine (idle)]:
created by addtimer
    /build/buildd/golang-1/src/pkg/runtime/ztime_amd64.c:69

goroutine 7 [runnable]:
syscall.Syscall()
    /build/buildd/golang-1/src/pkg/syscall/asm_linux_amd64.s:36 +0x64
syscall.Close(0xa, 0x0, 0x0, 0xf8400bd760)
    /usr/lib/go/src/pkg/syscall/zerrors_linux_amd64.go:1701 +0x54
os.(*file).close(0xf84007eea0, 0x46a301, 0x589740, 0x46b3a4)
    /usr/lib/go/src/pkg/os/file_unix.go:96 +0x4f
os.(*File).Close(0xf84008f650, 0x0, 0xf8400d6000, 0x46b4a3)
    /usr/lib/go/src/pkg/os/file_unix.go:88 +0x29
net.(*netFD).decref(0xf8400d6000, 0xf8400d6000)
    /usr/lib/go/src/pkg/net/fd.go:373 +0x66
net.(*netFD).Close(0xf8400d6000, 0x0, 0x0, 0x409e10)
    /usr/lib/go/src/pkg/net/fd.go:392 +0xde
net.(*TCPConn).Close(0xf84008f658, 0xf8401445f0, 0xf84019a001, 0x1)
    /usr/lib/go/src/pkg/net/tcpsock_posix.go:111 +0x93
github.com/4ad/doozer.(*Conn).mux(0xf8400c0140, 0xf8400d8060, 0x0, 0x0)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozer/conn.go:231 +0x38e
created by github.com/4ad/doozer.Dial
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozer/conn.go:57 +0x197

goroutine 12 [chan receive]:
github.com/4ad/doozerd/peer.activate(0xf8400d9310, 0xf8400bd960, 0x100000010, 0xf8400c0140, 0x1, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:230 +0x3b0
github.com/4ad/doozerd/peer._func_003(0xf84008f948, 0xf84008afa0, 0xf84008f8f0, 0xf84008f960, 0xf84008f550, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:131 +0x51
created by github.com/4ad/doozerd/peer.Main
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:149 +0x1410

goroutine 13 [chan receive]:
github.com/4ad/doozerd/member.Clean(0xf8400ea100, 0xf8400d9310, 0xf8400e9630, 0xf8400bdf60, 0x0, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/member/member.go:14 +0x58
created by github.com/4ad/doozerd/peer.Main
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:153 +0x8eb

goroutine 14 [chan receive]:
net.(*pollServer).WaitRead(0xf8400ba880, 0xf8400ac480, 0xf84007e840, 0xb, 0x1, ...)
    /usr/lib/go/src/pkg/net/fd.go:268 +0x73
net.(*netFD).accept(0xf8400ac480, 0x478b8e, 0x0, 0xf84007df00, 0xf84008f040, ...)
    /usr/lib/go/src/pkg/net/fd.go:622 +0x20d
net.(*TCPListener).AcceptTCP(0xf84008f388, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/lib/go/src/pkg/net/tcpsock_posix.go:322 +0x71
net.(*TCPListener).Accept(0xf84008f388, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/lib/go/src/pkg/net/tcpsock_posix.go:332 +0x49
github.com/4ad/doozerd/server.ListenAndServe(0xf8400ba900, 0xf84008f388, 0xf8400c0eb0, 0xf8400d9310, 0xf8400e9630, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/server/server.go:16 +0x42
created by github.com/4ad/doozerd/peer.Main
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:154 +0x9ac

goroutine 15 [chan receive]:
net.(*pollServer).WaitRead(0xf8400ba880, 0xf8400ac5a0, 0xf84007e840, 0xb, 0x1, ...)
    /usr/lib/go/src/pkg/net/fd.go:268 +0x73
net.(*netFD).accept(0xf8400ac5a0, 0x478b8e, 0x0, 0xf84007df00, 0xf84008f040, ...)
    /usr/lib/go/src/pkg/net/fd.go:622 +0x20d
net.(*TCPListener).AcceptTCP(0xf84008f410, 0xf8400ed280, 0x0, 0x0, 0x8, ...)
    /usr/lib/go/src/pkg/net/tcpsock_posix.go:322 +0x71
net.(*TCPListener).Accept(0xf84008f410, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/lib/go/src/pkg/net/tcpsock_posix.go:332 +0x49
net/http.(*Server).Serve(0xf84011e3c0, 0xf8400ba900, 0xf84008f410, 0x0, 0x0, ...)
    /usr/lib/go/src/pkg/net/http/server.go:1012 +0x88
net/http.Serve(0xf8400ba900, 0xf84008f410, 0x0, 0x0, 0x6ffc8c, ...)
    /usr/lib/go/src/pkg/net/http/server.go:977 +0x71
github.com/4ad/doozerd/web.Serve(0xf8400ba900, 0xf84008f410)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/web/web.go:46 +0x1a7
created by github.com/4ad/doozerd/peer.Main
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:159 +0xa2b

goroutine 16 [chan receive]:
github.com/4ad/doozerd/peer._func_004(0xf84008f940, 0xf84008f8f8, 0x0, 0x0)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:163 +0x46
created by github.com/4ad/doozerd/peer.Main
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/peer/peer.go:174 +0xa52

goroutine 18 [chan receive]:
github.com/4ad/doozerd/web.send(0xf8400eac00, 0xf84017c68c, 0xf800000001, 0xf8400f1460, 0xf84017b6a4, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/web/web.go:51 +0x59
github.com/4ad/doozerd/web._func_002(0xf8400edcf0, 0xf840111350, 0x5500de, 0xf8400eac00, 0xf8401112f0, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/web/web.go:97 +0x40
code.google.com/p/go.net/websocket.Handler.ServeHTTP(0xf84017b680, 0xf84017b200, 0xf84017b600, 0xf840101600, 0xf84017b680, ...)
    /home/bmurphy/workspace/go/src/pkg/code.google.com/p/go.net/websocket/server.go:76 +0x210
github.com/4ad/doozerd/web.evServer(0xf84017b200, 0xf84017b600, 0xf840101600, 0x4f9458, 0x526ce2, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/web/web.go:99 +0x204
net/http.HandlerFunc.ServeHTTP(0x526ce2, 0xf84017b200, 0xf84017b600, 0xf840101600, 0xf800000000, ...)
    /usr/lib/go/src/pkg/net/http/server.go:690 +0x3d
net/http.(*ServeMux).ServeHTTP(0xf84007a940, 0xf84017b200, 0xf84017b600, 0xf840101600, 0xf84017b600, ...)
    /usr/lib/go/src/pkg/net/http/server.go:924 +0xe7
net/http.(*conn).serve(0xf84007ca20, 0x0)
    /usr/lib/go/src/pkg/net/http/server.go:656 +0x621
created by net/http.(*Server).Serve
    /usr/lib/go/src/pkg/net/http/server.go:1040 +0x430

goroutine 19 [chan receive]:
github.com/4ad/doozerd/web._func_001(0xf8400edcf0, 0xf840111350, 0xf840111358, 0xf840111388, 0x0, ...)
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/web/web.go:86 +0xdc
created by github.com/4ad/doozerd/web.evServer
    /home/bmurphy/workspace/go/src/pkg/github.com/4ad/doozerd/web/web.go:94 +0x1a7

Any idea where I might be going wrong?

Thanks,
Bryan

Reply all

Reply to author

Forward

0 new messages