why makeSlice so expensive on memory usage?
davy zhang
package main
import (
"bytes"
"fmt"
"io"
"net"
"os"
"runtime"
"runtime/pprof"
)
func client(conn net.Conn, c chan int) {
data := make([]byte, 4096)
recvBuf := new(bytes.Buffer)
sndBuf := new(bytes.Buffer)
msgBuf := new(bytes.Buffer)
defer conn.Close()
for {
rn, err := conn.Read(data)
// fmt.Println("read size,err ", rn, ",", err)
switch err {
case io.EOF:
// fmt.Println("end of buff, quit... ", err)
// fmt.Println("routine count ",runtime.NumGoroutine())
runtime.GC()
return
case nil:
recvBuf.Write(data[:rn])
for {
line, err := recvBuf.ReadBytes('\n')
// fmt.Println("bytes recved:", string(line), err)
msgBuf.Write(line)
if err == io.EOF {
// fmt.Println("finally left ", string(line))
break
}
// fmt.Println("msg recved:",string(msgBuf.Bytes()))
sndBuf.Write(msgBuf.Bytes())
// wn, werr := sndBuf.WriteTo(conn)
sndBuf.WriteTo(conn)
// fmt.Println("written ", wn, werr)
msgBuf.Reset()
recvBuf.Reset()
sndBuf.Reset()
// fmt.Println("left ", len(recvBuf.Bytes()))
}
default:
fmt.Println("err:", err)
break
}
}
}
func svrListen(lis net.Listener, c chan int) {
for {
conn, err := lis.Accept()
if err != nil {
fmt.Errorf("accept error ", err)
}
// fmt.Println("accepted ", conn.RemoteAddr())
go client(conn, c)
}
}
func debugListen(lis net.Listener, c chan int) {
for {
conn, err := lis.Accept()
if err != nil {
fmt.Errorf("accept error ", err)
}
fmt.Println("debug accepted ", conn)
f, _ := os.Create("/tmp/memprof")
defer f.Close()
pprof.WriteHeapProfile(f)
pprof.StopCPUProfile()
conn.Close()
}
}
func cpuProfileStart() {
f, _ := os.Create("/tmp/server_cpu")
pprof.StartCPUProfile(f)
}
func main() {
var (
host = ""
port1 = "9000"
port2 = "8000"
c = make(chan int, 1)
)
lis1, _ := net.Listen("tcp", host+":"+port1)
lis2, _ := net.Listen("tcp", host+":"+port2)
lis3, _ := net.Listen("tcp", host+":6666")
go svrListen(lis1, c)
go svrListen(lis2, c)
go debugListen(lis3, c)
cpuProfileStart()
code := <-c
for {
if code == -1 {
break
}
}
}
test client code
package main
import (
"bufio"
"fmt"
"net"
"time"
"strings"
)
var addr string = "10.23.1.247:8000"
var debugAddr string = "10.23.1.247:6666"
//var addr string = "10.23.1.246:8000"
//var addr string = "localhost:8000"
func newConn(c chan int, packet string,count int) {
conn, err := net.Dial("tcp", addr)
defer conn.Close()
if err != nil {
panic(err)
}
rd := bufio.NewReadWriter(bufio.NewReader(conn), bufio.NewWriter(conn))
for i := 0; i < count; i++ {
rd.WriteString(packet)
rd.Flush()
rd.ReadString('\n')
}
// print(line)
c <- 1
}
func debugConn(){
conn,_ := net.Dial("tcp",debugAddr)
defer conn.Close()
}
func main() {
c := make(chan int, 1)
clientCount := 3000
repeatCount := 10
packet := strings.Repeat("0",1024 * 10)
packet += "\n"
for i := 0; i < clientCount; i++ {
go newConn(c,packet,repeatCount)
}
counter := 0
t1 := time.Now()
for ; counter < clientCount; {
counter += <-c
// print(counter)
}
fmt.Println(counter)
fmt.Println(time.Since(t1))
// debugConn()
}
here is the memory profile file I took in the middle of the runtime
heap profile: 88: 862800 [170: 1516240] @ heap/1048576
25: 366848 [75: 819456] @ 0x415907 0x41298d 0x412838 0x419315 0x418e05 0x418ee3 0x400e2d 0x40ee31
32: 335872 [47: 493312] @ 0x415907 0x41298d 0x412838 0x419315 0x418e05 0x418ee3 0x400efa 0x40ee31
13: 53248 [18: 73728] @ 0x415907 0x41298d 0x412838 0x400c3e 0x40ee31
1: 53248 [1: 53248] @ 0x415907 0x4164db 0x405250 0x405b58 0x40637b 0x406738 0x4072f9 0x40756a 0x4356f0 0x431e08 0x4327f5 0x43358a 0x440dcc 0x400cfe 0x40ee31
13: 53248 [17: 75776] @ 0x415907 0x41298d 0x412838 0x419315 0x418e05 0x418ee3 0x400dd4 0x40ee31
1: 96 [1: 96] @ 0x415907 0x416533 0x400c65 0x40ee31
1: 96 [1: 96] @ 0x415907 0x4164db 0x4034ab 0x403584 0x432ad7 0x434d2a 0x442246 0x442340 0x40106d 0x40ee31
1: 96 [1: 96] @ 0x415907 0x416533 0x400c7c 0x40ee31
1: 48 [9: 432] @ 0x415907 0x415abf 0x410970 0x433473 0x440dcc 0x400cfe 0x40ee31
Thanks for any hints
Russ Cox
Russ