Issue with CGO and GOGC when dealing with large buffers
Lealem Amedie
Hi,
I’m trying to get an https server working by overwriting the accept/read/write methods in the http module. I’m using go-wolfssl for TLS.
The server will accept a TLS 1.2 connection, then send some random data to the client connects, then wait for another connection.
The issue that I’m seeing is that whenever the server sends large payloads, it’ll do that successfully but it then can’t send data to the next client that connects (the send errors out with the TCP error EBADF). If dealing with smaller payloads (<30k), the server can send data successfully to each client that connects. Then as soon as a large one is sent, the next transmission fails.
If I disable the GOGC with debug.SetGCPercent(-1), the issue goes away and I can send as many large payloads as required. From the debugging I’ve done so far, this looks like an issue with the GO garbage collector dropping C pointers. go-wolfssl relies on the wolfSSL C library so it uses CGO. Does anyone have any other ideas or input?
Code is below. See this repo to actually run it GitHub - lealem47/go-wolfssl-https-server
Thank you!
```
package main
import (
"bytes"
"crypto/rand"
"encoding/base64"
"fmt"
log "github.com/sirupsen/logrus"
wolfSSL "github.com/wolfssl/go-wolfssl"
"net"
"net/http"
"os"
"strconv"
"sync"
"time"
)
const defaultPort = "8443"
type wolfSSLListener struct {
listener net.Listener
ctx *wolfSSL.WOLFSSL_CTX
}
// Accept waits for and returns the next connection to the listener.
func (cl *wolfSSLListener) Accept() (net.Conn, error) {
conn, err := cl.listener.Accept()
if err != nil {
return nil, err
}
fmt.Println("Accepted new connection from:", conn.RemoteAddr())
ssl := wolfSSL.WolfSSL_new(cl.ctx)
if ssl == nil {
fmt.Println("WolfSSL_new Failed")
os.Exit(1)
}
file, err := conn.(*net.TCPConn).File()
if err != nil {
panic(err)
}
fd := file.Fd()
wolfSSL.WolfSSL_set_fd(ssl, int(fd))
ret := wolfSSL.WolfSSL_accept(ssl)
if ret != wolfSSL.WOLFSSL_SUCCESS {
fmt.Println("WolfSSL_accept error ", ret)
} else {
fmt.Println("Client Successfully Connected!")
}
return &wolfSSLConn{
conn: conn,
ssl: ssl,
}, nil
}
// Close closes the listener, making it stop accepting new connections.
func (cl *wolfSSLListener) Close() error {
fmt.Println("Closing listener...")
return cl.listener.Close()
}
// Addr returns the listener's network address.
func (cl *wolfSSLListener) Addr() net.Addr {
return cl.listener.Addr()
}
type wolfSSLConn struct {
conn net.Conn
ssl *wolfSSL.WOLFSSL
buffer bytes.Buffer
mu sync.Mutex
closed bool
}
func (w *wolfSSLConn) Read(b []byte) (int, error) {
log.Infof("Calling read: %d", len(b))
ret := wolfSSL.WolfSSL_read(w.ssl, b, uintptr(len(b)))
if ret < 0 {
errCode := wolfSSL.WolfSSL_get_error(w.ssl, int(ret))
return 0, fmt.Errorf("read error: %d", errCode)
}
log.Infof("Read bytes: %s", string(b[:ret]))
return int(ret), nil
}
func (w *wolfSSLConn) Write(b []byte) (int, error) {
log.Infof("Calling write: %d", len(b))
sz := uintptr(len(b))
ret := wolfSSL.WolfSSL_write(w.ssl, b, sz)
if ret < 0 {
errCode := wolfSSL.WolfSSL_get_error(w.ssl, int(ret))
return 0, fmt.Errorf("write error: %d", errCode)
}
return int(ret), nil
}
func (w *wolfSSLConn) Close() error {
log.Infof("Closing connection")
wolfSSL.WolfSSL_shutdown(w.ssl)
wolfSSL.WolfSSL_free(w.ssl)
return w.conn.Close()
}
func (w *wolfSSLConn) LocalAddr() net.Addr {
return w.conn.LocalAddr()
}
func (w *wolfSSLConn) RemoteAddr() net.Addr {
return w.conn.RemoteAddr()
}
func (w *wolfSSLConn) SetDeadline(t time.Time) error {
return w.conn.SetDeadline(t)
}
func (w *wolfSSLConn) SetReadDeadline(t time.Time) error {
return w.conn.SetReadDeadline(t)
}
func (w *wolfSSLConn) SetWriteDeadline(t time.Time) error {
return w.conn.SetWriteDeadline(t)
}
// Handler for generating and base64 encoding 5KB of random data
func randomDataHandler(w http.ResponseWriter, r *http.Request) {
// Get the "size" query parameter from the request
sizeParam := r.URL.Query().Get("size")
size := 500000 // default size
// If the "size" parameter is provided, convert it to an integer
if sizeParam != "" {
parsedSize, err := strconv.Atoi(sizeParam)
if err != nil || parsedSize <= 0 {
http.Error(w, "Invalid size parameter", http.StatusBadRequest)
return
}
size = parsedSize
}
// Generate random data of the specified size
data := make([]byte, size)
_, err := rand.Read(data)
if err != nil {
http.Error(w, "Could not generate random data", http.StatusInternalServerError)
return
}
// Base64 encode the random data
encodedData := base64.StdEncoding.EncodeToString(data)
// Set content type and write the base64 encoded data
w.Header().Set("Content-Type", "application/base64")
w.Write([]byte(encodedData))
}
func main() {
port := defaultPort
// Set logging level
log.SetLevel(log.InfoLevel)
log.SetFormatter(&log.TextFormatter{
DisableColors: false,
FullTimestamp: true,
})
// Set up the HTTP server and routes
http.HandleFunc("/", randomDataHandler)
CERT_FILE := "./certs/server-cert.pem"
KEY_FILE := "./certs/server-key.pem"
/* Initialize wolfSSL */
wolfSSL.WolfSSL_Init()
/* Create WOLFSSL_CTX with tlsv12 */
ctx := wolfSSL.WolfSSL_CTX_new(wolfSSL.WolfTLSv1_2_server_method())
if ctx == nil {
fmt.Println(" WolfSSL_CTX_new Failed")
os.Exit(1)
}
/* Load server certificates into WOLFSSL_CTX */
ret := wolfSSL.WolfSSL_CTX_use_certificate_file(ctx, CERT_FILE, wolfSSL.SSL_FILETYPE_PEM)
if ret != wolfSSL.WOLFSSL_SUCCESS {
fmt.Println("Error: WolfSSL_CTX_use_certificate Failed")
os.Exit(1)
}
/* Load server key into WOLFSSL_CTX */
ret = wolfSSL.WolfSSL_CTX_use_PrivateKey_file(ctx, KEY_FILE, wolfSSL.SSL_FILETYPE_PEM)
if ret != wolfSSL.WOLFSSL_SUCCESS {
fmt.Println("Error: WolfSSL_CTX_use_PrivateKey Failed")
os.Exit(1)
}
baseListener, err := net.Listen("tcp", ":"+port)
if err != nil {
fmt.Println("Error starting listener:", err)
return
}
defer baseListener.Close()
wolfSSLListener := &wolfSSLListener{
listener: baseListener,
ctx: ctx,
}
log.Printf("Server listening on https://localhost:%s", port)
err = http.Serve(wolfSSLListener, nil)
if err != nil {
fmt.Println("Error starting HTTP server:", err)
}
}
```
robert engels
--
You received this message because you are subscribed to the Google Groups "golang-nuts" group.
To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts...@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/golang-nuts/a662824b-3555-4143-bfd8-b3431b7dc942n%40googlegroups.com.
robert engels
Ian Lance Taylor
>
> I’m trying to get an https server working by overwriting the accept/read/write methods in the http module. I’m using go-wolfssl for TLS.
>
> The server will accept a TLS 1.2 connection, then send some random data to the client connects, then wait for another connection.
>
> The issue that I’m seeing is that whenever the server sends large payloads, it’ll do that successfully but it then can’t send data to the next client that connects (the send errors out with the TCP error EBADF). If dealing with smaller payloads (<30k), the server can send data successfully to each client that connects. Then as soon as a large one is sent, the next transmission fails.
>
> If I disable the GOGC with debug.SetGCPercent(-1), the issue goes away and I can send as many large payloads as required. From the debugging I’ve done so far, this looks like an issue with the GO garbage collector dropping C pointers. go-wolfssl relies on the wolfSSL C library so it uses CGO. Does anyone have any other ideas or input?
>
> Code is below. See this repo to actually run it GitHub - lealem47/go-wolfssl-https-server
garbage collector, then it is most likely the case that the cgo code
is not following the rules described at
https://pkg.go.dev/cmd/cgo#hdr-Passing_pointers. You could try
setting GOEXPERIMENT=cgocheck2 in the environment when building the
program; that will enable stricter checks of cgo pointers, though it
still won't catch all possible misuses in the C code.
Ian
Ian Lance Taylor
>
> ChatGPT seems to think that the go-wolfssl library is not following proper CGO rules in
> many places. The transcript below is long. I post it nonetheless, in case such
> analysis is new for readers. I would suggest that you (everyone) can and should
> be doing this kind of "code review by LLM" yourself to quickly resolve
> these kinds of issues. Also, it might be wrong!?! since it comes from an LLM.
> But at least we can talk about specific code snippets now.
>
> Transcript using ChatGPT 4o from Github copilot:
>
> @workspace I suspect there are one or more bugs in the use of CGO in this this Go library, go-wolfssl. That is, I suspect that the CGO rules for passing C pointers back and forth between Go and C threads are not being followed, and this is causing bugs in the use of the library. Please audit all the Go and C files in this workspace, and point out places where the CGO pointer rules are not being followed.
>
> Workspace: In CGO, there are strict rules for passing pointers between Go and C. Specifically, Go pointers should not be passed to C code, and C pointers should not be used in Go code without proper conversion. Here are some potential violations of these rules in your workspace:
>
> 1. **`random.go`**:
> ```go
> func Wc_RNG_GenerateBlock(rng *C.struct_WC_RNG, b []byte, sz int) int {
> return int(C.wc_RNG_GenerateBlock(rng, (*C.uchar)(unsafe.Pointer(&b[0])), C.word32(sz)))
> }
> ```
> - The Go slice
>
> b
>
> is being converted to a C pointer and passed to the C function. This is a violation of the CGO rules.
This code is in itself fine, and is a typical example of using cgo.
I didn't read any further.
Ian
Amnon
Why not just make life easy for yourself by sticking to the http.ServeTLS built in to the Go standard library.
Jason E. Aten
robert engels
On Dec 2, 2024, at 4:43 PM, Jason E. Aten <j.e....@gmail.com> wrote:
Sorry. I've deleted those unhelpful suggestions.
--
You received this message because you are subscribed to the Google Groups "golang-nuts" group.
To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts...@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/golang-nuts/455959a3-4ddf-42b9-9d06-4d651effd7e2n%40googlegroups.com.
robert engels
Jason E. Aten
INFO[2024-12-03T15:02:51-06:00] Server listening on https://localhost:8443
INFO[2024-12-03T15:03:37-06:00] Read bytes: GET /?size=5000000 HTTP/1.1
Host: localhost:8443
User-Agent: curl/7.58.0
Accept: */*
INFO[2024-12-03T15:03:37-06:00] Calling read: 1
INFO[2024-12-03T15:03:37-06:00] Calling write: 4096
INFO[2024-12-03T15:03:37-06:00] Closing connection
INFO[2024-12-03T15:03:37-06:00] Closing connection
free(): invalid pointer
SIGABRT: abort
PC=0x7f7189772e87 m=0 sigcode=18446744073709551610
signal arrived during cgo execution
goroutine 34 gp=0xc000104c40 m=0 mp=0xb00ea0 [syscall]:
runtime.cgocall(0x656db0, 0xc0002179d0)
/usr/local/go1.23.3/src/runtime/cgocall.go:167 +0x4b fp=0xc0002179a8 sp=0xc000217970 pc=0x46ab8b
github.com/wolfssl/go-wolfssl._Cfunc_wolfSSL_free(0x7f712c000b20)
_cgo_gotypes.go:1598 +0x3f fp=0xc0002179d0 sp=0xc0002179a8 pc=0x50121f
main.(*wolfSSLConn).Close.WolfSSL_free.func2(0x7f712c000b20)
/home/jaten/go/pkg/mod/github.com/wolfssl/go-wo...@v0.0.0-20240829213546-44165fae06e4/ssl.go:113 +0x3b fp=0xc000217a08 sp=0xc0002179d0 pc=0x654f9b
github.com/wolfssl/go-wolfssl.WolfSSL_free(...)
/home/jaten/go/pkg/mod/github.com/wolfssl/go-wo...@v0.0.0-20240829213546-44165fae06e4/ssl.go:113
main.(*wolfSSLConn).Close(0xc0001a4000)
/home/jaten/go/src/github.com/lealem47/go-wolfssl-https-server/cmd/rest/main.go:109 +0x58 fp=0xc000217a50 sp=0xc000217a08 pc=0x654f18
net/http.(*conn).close(0xc0001a6000)
/usr/local/go1.23.3/src/net/http/server.go:1800 +0x2b fp=0xc000217a68 sp=0xc000217a50 pc=0x6404ab
net/http.(*conn).serve.func1()
/usr/local/go1.23.3/src/net/http/server.go:1959 +0x1e8 fp=0xc000217b08 sp=0xc000217a68 pc=0x641e68
runtime.deferreturn()
/usr/local/go1.23.3/src/runtime/panic.go:605 +0x5e fp=0xc000217b98 sp=0xc000217b08 pc=0x438d3e
net/http.(*conn).serve(0xc0001a6000, {0x74e0b0, 0xc000128ff0})
/usr/local/go1.23.3/src/net/http/server.go:2104 +0x7aa fp=0xc000217fb8 sp=0xc000217b98 pc=0x64108a
net/http.(*Server).Serve.gowrap3()
/usr/local/go1.23.3/src/net/http/server.go:3360 +0x28 fp=0xc000217fe0 sp=0xc000217fb8 pc=0x6457e8
runtime.goexit({})
/usr/local/go1.23.3/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc000217fe8 sp=0xc000217fe0 pc=0x477fc1
created by net/http.(*Server).Serve in goroutine 1
/usr/local/go1.23.3/src/net/http/server.go:3360 +0x485
goroutine 1 gp=0xc0000061c0 m=nil [IO wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/local/go1.23.3/src/runtime/proc.go:424 +0xce fp=0xc0000f99e8 sp=0xc0000f99c8 pc=0x47056e
runtime.netpollblock(0xc000035a40?, 0x409626?, 0x0?)
/usr/local/go1.23.3/src/runtime/netpoll.go:575 +0xf7 fp=0xc0000f9a20 sp=0xc0000f99e8 pc=0x4358f7
internal/poll.runtime_pollWait(0x7f7140dafe00, 0x72)
/usr/local/go1.23.3/src/runtime/netpoll.go:351 +0x85 fp=0xc0000f9a40 sp=0xc0000f9a20 pc=0x46f865
internal/poll.(*pollDesc).wait(0xc000176000?, 0x50?, 0x0)
/usr/local/go1.23.3/src/internal/poll/fd_poll_runtime.go:84 +0x27 fp=0xc0000f9a68 sp=0xc0000f9a40 pc=0x4b1147
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go1.23.3/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc000176000)
/usr/local/go1.23.3/src/internal/poll/fd_unix.go:620 +0x295 fp=0xc0000f9b10 sp=0xc0000f9a68 pc=0x4b2875
net.(*netFD).accept(0xc000176000)
/usr/local/go1.23.3/src/net/fd_unix.go:172 +0x29 fp=0xc0000f9bc8 sp=0xc0000f9b10 pc=0x51cd49
net.(*TCPListener).accept(0xc000132240)
/usr/local/go1.23.3/src/net/tcpsock_posix.go:159 +0x1e fp=0xc0000f9c18 sp=0xc0000f9bc8 pc=0x52d23e
net.(*TCPListener).Accept(0xc000132240)
/usr/local/go1.23.3/src/net/tcpsock.go:372 +0x30 fp=0xc0000f9c48 sp=0xc0000f9c18 pc=0x52c570
main.(*wolfSSLListener).Accept(0xc000126198)
/home/jaten/go/src/github.com/lealem47/go-wolfssl-https-server/cmd/rest/main.go:27 +0x32 fp=0xc0000f9d00 sp=0xc0000f9c48 pc=0x654392
net/http.(*onceCloseListener).Accept(0xc0001a6000?)
<autogenerated>:1 +0x24 fp=0xc0000f9d18 sp=0xc0000f9d00 pc=0x6536a4
net/http.(*Server).Serve(0xc0001640f0, {0x74da88, 0xc000126198})
/usr/local/go1.23.3/src/net/http/server.go:3330 +0x30c fp=0xc0000f9e48 sp=0xc0000f9d18 pc=0x6453ec
net/http.Serve(...)
/usr/local/go1.23.3/src/net/http/server.go:2858
main.main()
/home/jaten/go/src/github.com/lealem47/go-wolfssl-https-server/cmd/rest/main.go:219 +0x392 fp=0xc0000f9f50 sp=0xc0000f9e48 pc=0x655792
runtime.main()
/usr/local/go1.23.3/src/runtime/proc.go:272 +0x28b fp=0xc0000f9fe0 sp=0xc0000f9f50 pc=0x43ce8b
runtime.goexit({})
/usr/local/go1.23.3/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc0000f9fe8 sp=0xc0000f9fe0 pc=0x477fc1
...
Robert Engels
On Dec 3, 2024, at 3:31 PM, Jason E. Aten <j.e....@gmail.com> wrote:
Lealem, I wasn't able to reproduce the error you were seeing (perhaps write a test case that does?), but when
--
You received this message because you are subscribed to the Google Groups "golang-nuts" group.
To unsubscribe from this group and stop receiving emails from it, send an email to golang-nuts...@googlegroups.com.
To view this discussion visit https://groups.google.com/d/msgid/golang-nuts/ec0aaa89-23a4-45be-8626-fb6eaf8fd4bdn%40googlegroups.com.
Lealem Amedie
p.Pin(file)
defer p.Unpin()
robert engels
Jason E. Aten
Jason