internal/runtime/atomic: add Cas128 primitive
This is the initial work for the upcoming atomic.Uint64Pair.
amd64 uses LOCK CMPXCHG16B; arm64 uses an LDAXP/STLXP loop; all
other architectures fall back to a small fixed lock table keyed
by the 16-byte-aligned pair address.
No GOAMD64 or GOARM64 feature gating for now. There are no
callers yet, so this is safe.
For https://github.com/golang/go/issues/61236
diff --git a/src/internal/runtime/atomic/atomic_amd64.go b/src/internal/runtime/atomic/atomic_amd64.go
index 84366ff..b29731f 100644
--- a/src/internal/runtime/atomic/atomic_amd64.go
+++ b/src/internal/runtime/atomic/atomic_amd64.go
@@ -104,6 +104,14 @@
//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool
+// Cas128 atomically compares the 128 bits at *ptr to (old1, old2) and, if
+// equal, replaces them with (new1, new2), returning whether the swap
+// happened. The two halves correspond to consecutive uint64s in memory:
+// (*ptr, *(ptr+1)). ptr must be 16-byte aligned.
+//
+//go:noescape
+func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool
+
//go:noescape
func CasRel(ptr *uint32, old, new uint32) bool
diff --git a/src/internal/runtime/atomic/atomic_amd64.s b/src/internal/runtime/atomic/atomic_amd64.s
index 301725e..24847fb 100644
--- a/src/internal/runtime/atomic/atomic_amd64.s
+++ b/src/internal/runtime/atomic/atomic_amd64.s
@@ -70,6 +70,32 @@
SETEQ ret+24(FP)
RET
+// func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool
+// Atomically:
+// if *ptr == old1 && *(ptr+1) == old2 {
+// *ptr = new1
+// *(ptr+1) = new2
+// return true
+// } else {
+// return false
+// }
+//
+// CMPXCHG16B requires its memory operand to be 16-byte aligned;
+// unaligned accesses fault.
+TEXT ·Cas128(SB), NOSPLIT, $0-41
+ MOVQ ptr+0(FP), DI
+ TESTQ $15, DI
+ JZ 2(PC)
+ CALL ·panicUnaligned128(SB)
+ MOVQ old1+8(FP), AX
+ MOVQ old2+16(FP), DX
+ MOVQ new1+24(FP), BX
+ MOVQ new2+32(FP), CX
+ LOCK
+ CMPXCHG16B (DI)
+ SETEQ ret+40(FP)
+ RET
+
TEXT ·Casint32(SB), NOSPLIT, $0-17
JMP ·Cas(SB)
diff --git a/src/internal/runtime/atomic/atomic_arm64.go b/src/internal/runtime/atomic/atomic_arm64.go
index f4aef19..d5efcc0 100644
--- a/src/internal/runtime/atomic/atomic_arm64.go
+++ b/src/internal/runtime/atomic/atomic_arm64.go
@@ -90,6 +90,14 @@
//go:noescape
func Cas64(ptr *uint64, old, new uint64) bool
+// Cas128 atomically compares the 128 bits at *ptr to (old1, old2) and, if
+// equal, replaces them with (new1, new2), returning whether the swap
+// happened. The two halves correspond to consecutive uint64s in memory:
+// (*ptr, *(ptr+1)). ptr must be 16-byte aligned.
+//
+//go:noescape
+func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool
+
//go:noescape
func CasRel(ptr *uint32, old, new uint32) bool
diff --git a/src/internal/runtime/atomic/atomic_arm64.s b/src/internal/runtime/atomic/atomic_arm64.s
index 360f7a2..764af8a 100644
--- a/src/internal/runtime/atomic/atomic_arm64.s
+++ b/src/internal/runtime/atomic/atomic_arm64.s
@@ -262,6 +262,41 @@
RET
#endif
+// func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool
+// Atomically:
+// if *ptr == old1 && *(ptr+1) == old2 {
+// *ptr = new1
+// *(ptr+1) = new2
+// return true
+// } else {
+// return false
+// }
+//
+// LDAXP/STLXP requires its memory operand to be 16-byte aligned;
+// unaligned accesses fault.
+TEXT ·Cas128(SB), NOSPLIT, $0-41
+ MOVD ptr+0(FP), R0
+ AND $15, R0, R7
+ CBZ R7, aligned
+ CALL ·panicUnaligned128(SB)
+aligned:
+ MOVD old1+8(FP), R1
+ MOVD old2+16(FP), R2
+ MOVD new1+24(FP), R3
+ MOVD new2+32(FP), R4
+load_store_loop:
+ LDAXP (R0), (R5, R6)
+ CMP R1, R5
+ BNE done
+ CMP R2, R6
+ BNE done
+ STLXP (R3, R4), (R0), R7
+ CBNZ R7, load_store_loop
+done:
+ CSET EQ, R0
+ MOVB R0, ret+40(FP)
+ RET
+
// uint32 xadd(uint32 volatile *ptr, int32 delta)
// Atomically:
// *val += delta;
diff --git a/src/internal/runtime/atomic/atomic_cas128_generic.go b/src/internal/runtime/atomic/atomic_cas128_generic.go
new file mode 100644
index 0000000..673f743
--- /dev/null
+++ b/src/internal/runtime/atomic/atomic_cas128_generic.go
@@ -0,0 +1,63 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !amd64 && !arm64
+
+package atomic
+
+import (
+ "internal/cpu"
+ "unsafe"
+)
+
+// Cas128 atomically compares the 128 bits at *ptr to (old1, old2) and, if
+// equal, replaces them with (new1, new2). Without a native 128-bit atomic
+// instruction, this falls back to a small fixed lock table keyed by the
+// 16-byte-aligned address of the pair. ptr must be 16-byte aligned.
+//
+//go:nosplit
+func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool {
+ if uintptr(unsafe.Pointer(ptr))&15 != 0 {
+ panicUnaligned128()
+ }
+ _ = *ptr // fault on nil before taking the lock
+ pair := (*[2]uint64)(unsafe.Pointer(ptr))
+ l := pairAddrLock(ptr)
+ l.lock()
+ ok := false
+ if pair[0] == old1 && pair[1] == old2 {
+ pair[0] = new1
+ pair[1] = new2
+ ok = true
+ }
+ l.unlock()
+ return ok
+}
+
+type pairSpinlock struct {
+ v uint32
+}
+
+//go:nosplit
+func (l *pairSpinlock) lock() {
+ for !Cas(&l.v, 0, 1) {
+ }
+}
+
+//go:nosplit
+func (l *pairSpinlock) unlock() {
+ Store(&l.v, 0)
+}
+
+// pairLocktab is a small open-addressed table of spinlocks keyed by the
+// 16-byte-aligned address of the pair. Sized prime to spread collisions.
+var pairLocktab [57]struct {
+ l pairSpinlock
+ pad [cpu.CacheLinePadSize - unsafe.Sizeof(pairSpinlock{})]byte
+}
+
+//go:nosplit
+func pairAddrLock(addr *uint64) *pairSpinlock {
+ return &pairLocktab[(uintptr(unsafe.Pointer(addr))>>4)%uintptr(len(pairLocktab))].l
+}
diff --git a/src/internal/runtime/atomic/atomic_pair_test.go b/src/internal/runtime/atomic/atomic_pair_test.go
new file mode 100644
index 0000000..4ad6ce1
--- /dev/null
+++ b/src/internal/runtime/atomic/atomic_pair_test.go
@@ -0,0 +1,115 @@
+// Copyright 2026 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package atomic_test
+
+import (
+ "internal/runtime/atomic"
+ "testing"
+ "unsafe"
+)
+
+// alignedPair returns a 16-byte-aligned *uint64 pointing inside buf. buf
+// must be at least 3 uint64s wide so a 16-byte-aligned 16-byte region fits.
+func alignedPair(buf *[3]uint64) *uint64 {
+ p := (uintptr(unsafe.Pointer(&buf[0])) + 15) &^ 15
+ return (*uint64)(unsafe.Pointer(p))
+}
+
+func TestCas128(t *testing.T) {
+ var buf [3]uint64
+ addr := alignedPair(&buf)
+ pair := (*[2]uint64)(unsafe.Pointer(addr))
+
+ // Successful CAS from (0, 0) to (1, 2).
+ if !atomic.Cas128(addr, 0, 0, 1, 2) {
+ t.Fatal("Cas128: should have succeeded from zero")
+ }
+ if pair[0] != 1 || pair[1] != 2 {
+ t.Fatalf("Cas128 corrupt write: got (%d, %d), want (1, 2)", pair[0], pair[1])
+ }
+
+ // Mismatch on low half: should fail without writing.
+ if atomic.Cas128(addr, 0, 2, 9, 9) {
+ t.Fatal("Cas128: should have failed on low-half mismatch")
+ }
+ // Mismatch on high half: should fail without writing.
+ if atomic.Cas128(addr, 1, 0, 9, 9) {
+ t.Fatal("Cas128: should have failed on high-half mismatch")
+ }
+ if pair[0] != 1 || pair[1] != 2 {
+ t.Fatalf("Cas128 wrote on failed CAS: got (%d, %d), want (1, 2)", pair[0], pair[1])
+ }
+
+ // Concurrent test: 32 goroutines each bump (lo, hi) -> (lo+1, hi-1)
+ // 1000 times. The invariant lo + hi == initialHi holds iff every
+ // successful CAS updated both halves together.
+ const initialHi = uint64(0xdeadbeefcafebabe)
+ pair[0], pair[1] = 0, initialHi
+
+ const G, N = 32, 1000
+ done := make(chan struct{})
+ for g := 0; g < G; g++ {
+ go func() {
+ for i := 0; i < N; i++ {
+ for {
+ lo := atomic.Load64(&pair[0])
+ hi := atomic.Load64(&pair[1])
+ if atomic.Cas128(addr, lo, hi, lo+1, hi-1) {
+ break
+ }
+ }
+ }
+ done <- struct{}{}
+ }()
+ }
+ for g := 0; g < G; g++ {
+ <-done
+ }
+ if got, want := atomic.Load64(&pair[0]), uint64(G*N); got != want {
+ t.Errorf("low half: got %d, want %d", got, want)
+ }
+ if got, want := atomic.Load64(&pair[1]), initialHi-uint64(G*N); got != want {
+ t.Errorf("high half: got %#x, want %#x", got, want)
+ }
+}
+
+func TestCas128Unaligned(t *testing.T) {
+ var buf [3]uint64
+ aligned := alignedPair(&buf)
+ // One uint64 past the aligned slot is 8-aligned but not 16-aligned.
+ misaligned := (*uint64)(unsafe.Pointer(uintptr(unsafe.Pointer(aligned)) + 8))
+
+ defer func() {
+ err := recover()
+ const want = "unaligned 128-bit atomic operation"
+ if err == nil {
+ t.Fatal("Cas128 on misaligned address did not panic")
+ }
+ if s, _ := err.(string); s != want {
+ t.Fatalf("Cas128: got panic %q, want %q", err, want)
+ }
+ }()
+ atomic.Cas128(misaligned, 0, 0, 0, 0)
+}
+
+func BenchmarkCas128(b *testing.B) {
+ var buf [3]uint64
+ addr := alignedPair(&buf)
+ sink = addr
+ for i := 0; i < b.N; i++ {
+ atomic.Cas128(addr, 0, 0, 0, 0)
+ }
+}
+
+func BenchmarkCas128Parallel(b *testing.B) {
+ var buf [3]uint64
+ addr := alignedPair(&buf)
+ sink = addr
+ b.RunParallel(func(pb *testing.PB) {
+ for pb.Next() {
+ atomic.Cas128(addr, 0, 0, 0, 0)
+ }
+ })
+}
diff --git a/src/internal/runtime/atomic/unaligned.go b/src/internal/runtime/atomic/unaligned.go
index a859de4..512ae3f 100644
--- a/src/internal/runtime/atomic/unaligned.go
+++ b/src/internal/runtime/atomic/unaligned.go
@@ -7,3 +7,7 @@
func panicUnaligned() {
panic("unaligned 64-bit atomic operation")
}
+
+func panicUnaligned128() {
+ panic("unaligned 128-bit atomic operation")
+}
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
I spotted some possible problems with your PR:
1. You usually need to reference a bug number for all but trivial or cosmetic fixes. For this repo, the format is usually 'Fixes #12345' or 'Updates #12345' at the end of the commit message. Should you have a bug reference?
Please address any problems by updating the GitHub PR.
When complete, mark this comment as 'Done' and click the [blue 'Reply' button](https://go.dev/wiki/GerritBot#i-left-a-reply-to-a-comment-in-gerrit-but-no-one-but-me-can-see-it) above. These findings are based on heuristics; if a finding does not apply, briefly reply here saying so.
To update the commit title or commit message body shown here in Gerrit, you must edit the GitHub PR title and PR description (the first comment) in the GitHub web interface using the 'Edit' button or 'Edit' menu entry there. Note: pushing a new commit to the PR will not automatically update the commit message used by Gerrit.
For more details, see:
(In general for Gerrit code reviews, the change author is expected to [log in to Gerrit](https://go-review.googlesource.com/login/) with a Gmail or other Google account and then close out each piece of feedback by marking it as 'Done' if implemented as suggested or otherwise reply to each review comment. See the [Review](https://go.dev/doc/contribute#review) section of the Contributing Guide for details.)
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Commit-Queue | +1 |
I spotted some possible problems with your PR:
1. You usually need to reference a bug number for all but trivial or cosmetic fixes. For this repo, the format is usually 'Fixes #12345' or 'Updates #12345' at the end of the commit message. Should you have a bug reference?Please address any problems by updating the GitHub PR.
When complete, mark this comment as 'Done' and click the [blue 'Reply' button](https://go.dev/wiki/GerritBot#i-left-a-reply-to-a-comment-in-gerrit-but-no-one-but-me-can-see-it) above. These findings are based on heuristics; if a finding does not apply, briefly reply here saying so.
To update the commit title or commit message body shown here in Gerrit, you must edit the GitHub PR title and PR description (the first comment) in the GitHub web interface using the 'Edit' button or 'Edit' menu entry there. Note: pushing a new commit to the PR will not automatically update the commit message used by Gerrit.
For more details, see:
- [how to update commit messages](https://go.dev/wiki/GerritBot/#how-does-gerritbot-determine-the-final-commit-message) for PRs imported into Gerrit.
- the Go project's [conventions for commit messages](https://go.dev/doc/contribute#commit_messages) that you should follow.
(In general for Gerrit code reviews, the change author is expected to [log in to Gerrit](https://go-review.googlesource.com/login/) with a Gmail or other Google account and then close out each piece of feedback by marking it as 'Done' if implemented as suggested or otherwise reply to each review comment. See the [Review](https://go.dev/doc/contribute#review) section of the Contributing Guide for details.)
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
LUCI failure seems unrelated.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool {Why not make the API memory safe ? `ptr *[2]uint64` makes more sense.
I think it would also look better with `old, new [2]uint64` but I don't care which way this goes.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool {Why not make the API memory safe ? `ptr *[2]uint64` makes more sense.
I think it would also look better with `old, new [2]uint64` but I don't care which way this goes.
Agree, but looking at Cas/Cas64, it makes more sense to just follow the same signature for the new function.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool {Mauri de Souza MeneguzzoWhy not make the API memory safe ? `ptr *[2]uint64` makes more sense.
I think it would also look better with `old, new [2]uint64` but I don't care which way this goes.
Agree, but looking at Cas/Cas64, it makes more sense to just follow the same signature for the new function.
Cas64 takes a `*uint64` pointer which is the size of the element it'll read / write to.
I truelly think `unsafe.Sizeof(*ptr)` needs to be `16` or bigger.
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool {Mauri de Souza MeneguzzoWhy not make the API memory safe ? `ptr *[2]uint64` makes more sense.
I think it would also look better with `old, new [2]uint64` but I don't care which way this goes.
JorropoAgree, but looking at Cas/Cas64, it makes more sense to just follow the same signature for the new function.
Cas64 takes a `*uint64` pointer which is the size of the element it'll read / write to.
I truelly think `unsafe.Sizeof(*ptr)` needs to be `16` or bigger.
Acknowledged
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
func Cas128(ptr *uint64, old1, old2, new1, new2 uint64) bool {Mauri de Souza MeneguzzoWhy not make the API memory safe ? `ptr *[2]uint64` makes more sense.
I think it would also look better with `old, new [2]uint64` but I don't care which way this goes.
JorropoAgree, but looking at Cas/Cas64, it makes more sense to just follow the same signature for the new function.
Mauri de Souza MeneguzzoCas64 takes a `*uint64` pointer which is the size of the element it'll read / write to.
I truelly think `unsafe.Sizeof(*ptr)` needs to be `16` or bigger.
Acknowledged
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
CMPXCHG16B (DI)TODO: This is GOAMD64=v2+, so we need a fallback for v1
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
| Commit-Queue | +1 |
TODO: This is GOAMD64=v2+, so we need a fallback for v1
Done
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |