[go/dev.simd] [dev.simd] runtime: use scan kernels in scanSpan [green tea]
4 views
Skip to first unread message
Junyang Shao (Gerrit)
Sep 11, 2025, 11:20:43 AMSep 11
to goph...@pubsubhelper.golang.org, golang-co...@googlegroups.com
Junyang Shao has uploaded the change for review![]( "Open in Gerrit")
Commit message
[dev.simd] runtime: use scan kernels in scanSpan [green tea]
This is an extra 15-20% faster over the current sparse span scanning
when AVX512+GFNI is available and there's sufficient density.
For #73581.
Copy of original CL
Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64_avx512,gotip-linux-amd64_avx512-greenteagc
Change-Id: I5f82a8d822a9cf056a953303ccfa6be7a7330dcd
Change diff
diff --git a/src/runtime/mgcmark_greenteagc.go b/src/runtime/mgcmark_greenteagc.go
index 845857a..55bb2cd 100644
--- a/src/runtime/mgcmark_greenteagc.go
+++ b/src/runtime/mgcmark_greenteagc.go
@@ -41,6 +41,7 @@
"internal/goarch"
"internal/runtime/atomic"
"internal/runtime/gc"
+ "internal/runtime/gc/scan"
"internal/runtime/sys"
"unsafe"
)
@@ -259,7 +260,7 @@
return heapBitsInSpan(size) && size >= 16
}
-// tryQueueOnSpan tries to queue p on the span it points to, if it
+// tryDeferToSpanScan tries to queue p on the span it points to, if it
// points to a small object span (gcUsesSpanQueue size).
func tryDeferToSpanScan(p uintptr, gcw *gcWork) bool {
if useCheckmark {
@@ -608,8 +609,7 @@
atomic.Or8(bytep, mask)
gcw.bytesMarked += uint64(elemsize)
if debug.gctrace > 1 {
- gcw.stats[spanclass.sizeclass()].spansSparseScanned++
- gcw.stats[spanclass.sizeclass()].spanObjsSparseScanned++
+ gcw.stats[spanclass.sizeclass()].sparseObjsScanned++
}
b := spanBase + uintptr(objIndex)*elemsize
scanObjectSmall(spanBase, b, elemsize, gcw)
@@ -631,11 +631,47 @@
return
}
gcw.bytesMarked += uint64(objsMarked) * uint64(elemsize)
- if debug.gctrace > 1 {
- gcw.stats[spanclass.sizeclass()].spansDenseScanned++
- gcw.stats[spanclass.sizeclass()].spanObjsDenseScanned += uint64(objsMarked)
+
+ // Check if we have enough density to make a dartboard scan
+ // worthwhile. If not, just do what scanobject does, but
+ // localized to the span, using the dartboard.
+ if !scan.HasFastScanSpanPacked() || objsMarked < int(nelems/8) {
+ if debug.gctrace > 1 {
+ gcw.stats[spanclass].spansSparseScanned++
+ gcw.stats[spanclass].spanObjsSparseScanned += uint64(objsMarked)
+ }
+ scanObjectsSmall(spanBase, elemsize, nelems, gcw, &toScan)
+ return
}
- scanObjectsSmall(spanBase, elemsize, nelems, gcw, &toScan)
+
+ // Scan the span.
+ //
+ // N.B. Use gcw.ptrBuf as the output buffer. This is a bit different
+ // from scanObjectsSmall, which puts addresses to dereference. ScanSpanPacked
+ // on the other hand, fills gcw.ptrBuf with already dereferenced pointers.
+ nptrs := scan.ScanSpanPacked(
+ unsafe.Pointer(spanBase),
+ &gcw.ptrBuf[0],
+ &toScan,
+ uintptr(spanclass.sizeclass()),
+ spanPtrMaskUnsafe(spanBase),
+ )
+ gcw.heapScanWork += int64(objsMarked) * int64(elemsize)
+
+ if debug.gctrace > 1 {
+ // Write down some statistics.
+ gcw.stats[spanclass].spansDenseScanned++
+ gcw.stats[spanclass].spanObjsDenseScanned += uint64(objsMarked)
+ }
+
+ // Process all the pointers we just got.
+ for _, p := range gcw.ptrBuf[:nptrs] {
+ if !tryDeferToSpanScan(p, gcw) {
+ if obj, span, objIndex := findObject(p, 0, 0); obj != 0 {
+ greyobject(obj, 0, 0, span, gcw, objIndex)
+ }
+ }
+ }
}
// spanSetScans sets any unset mark bits that have their mark bits set in the inline mark bits.
@@ -798,12 +834,27 @@
return read
}
+// spanPtrMaskUnsafe returns the pointer mask for a span with inline mark bits.
+//
+// The caller must ensure spanBase is the base of a span that:
+// - 1 page in size,
+// - Uses inline mark bits,
+// - Contains pointers.
+func spanPtrMaskUnsafe(spanBase uintptr) *gc.PtrMask {
+ base := spanBase + gc.PageSize - unsafe.Sizeof(gc.PtrMask{}) - unsafe.Sizeof(spanInlineMarkBits{})
+ return (*gc.PtrMask)(unsafe.Pointer(base))
+}
+
type sizeClassScanStats struct {
- spansDenseScanned uint64
- spanObjsDenseScanned uint64
- spansSparseScanned uint64
- spanObjsSparseScanned uint64
- sparseObjsScanned uint64
+ spansDenseScanned uint64 // Spans scanned with ScanSpanPacked.
+ spanObjsDenseScanned uint64 // Objects scanned with ScanSpanPacked.
+ spansSparseScanned uint64 // Spans scanned with scanObjectsSmall.
+ spanObjsSparseScanned uint64 // Objects scanned with scanObjectsSmall.
+ sparseObjsScanned uint64 // Objects scanned with scanobject or scanObjectSmall.
+ // Note: sparseObjsScanned is sufficient for both cases because
+ // a particular size class either uses scanobject or scanObjectSmall,
+ // not both. In the latter case, we also know that there was one
+ // object scanned per span, so no need for a span counter.
}
func dumpScanStats() {
Change information
Files:
- M src/runtime/mgcmark_greenteagc.go
Change size: M
Delta: 1 file changed, 63 insertions(+), 12 deletions(-)
Related details
Attention set is empty
Submit Requirements:
Code-Review
No-Unresolved-Comments
Review-Enforcement
TryBots-Pass
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Junyang Shao (Gerrit)
Sep 11, 2025, 11:25:47 AMSep 11
to goph...@pubsubhelper.golang.org, golang-co...@googlegroups.com
Junyang Shao uploaded new patchset![]( "Open in Gerrit")
Junyang Shao uploaded patch set #3 to this change.
Related details
Attention set is empty
Submit Requirements:
Code-Review
No-Unresolved-Comments
Review-Enforcement
TryBots-Pass
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Junyang Shao (Gerrit)
Sep 11, 2025, 11:26:07 AMSep 11
to goph...@pubsubhelper.golang.org, golang-co...@googlegroups.com
Junyang Shao voted Commit-Queue+1![]( "Open in Gerrit")
| Commit-Queue | +1 |
Related details
Attention set is empty
Submit Requirements:
Code-Review
No-Unresolved-Comments
Review-Enforcement
TryBots-Pass
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Junyang Shao (Gerrit)
Sep 11, 2025, 2:37:16 PMSep 11
to goph...@pubsubhelper.golang.org, golang-co...@googlegroups.com
Junyang Shao uploaded new patchset![]( "Open in Gerrit")
Junyang Shao uploaded patch set #4 to this change.
Following approvals got outdated and were removed:
- TryBots-Pass: LUCI-TryBot-Result+1 by Go LUCI
Related details
Attention set is empty
Submit Requirements:
Code-Review
No-Unresolved-Comments
Review-Enforcement
TryBots-Pass
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Junyang Shao (Gerrit)
Sep 12, 2025, 12:47:14 PMSep 12
to goph...@pubsubhelper.golang.org, golang-co...@googlegroups.com
Junyang Shao uploaded new patchset![]( "Open in Gerrit")
Junyang Shao uploaded patch set #7 to this change.
Related details
Attention set is empty
Submit Requirements:
Code-Review
No-Unresolved-Comments
Review-Enforcement
TryBots-Pass
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Reply all
Reply to author
Forward
0 new messages