[go] cmd/compile: use RVV for RISCV64 LoweredMoveLoop
2 views
Skip to first unread message
Julian Zhu (Gerrit)
Jul 16, 2026, 1:29:59 PM (10 days ago) Jul 16
to goph...@pubsubhelper.golang.org, golang-co...@googlegroups.com
Julian Zhu has uploaded the change for review![]( "Open in Gerrit")
Commit message
cmd/compile: use RVV for RISCV64 LoweredMoveLoop
Use an RVV move loop for large Move operations on all RISC-V profiles,
with a runtime dispatch on riscv64HasV.
For GORISCV64 >= rva23, the compiler emits LoweredMoveLoopV directly,
which always runs the RVV loop.
For GORISCV64 < rva23, the compiler emits LoweredMoveLoop, and will
perform a runtime check of riscv64HasV for large Move operations.
The RVV loop uses V24-V31 as an LMUL=8 group (via VS8R/VL8R), X5 as
the remaining byte count, and X6 as the active VL. It sets VL with
AVL=n before the main loop, so the hot loop has no vsetvli instruction;
only a non-empty tail requires a second vsetvli.
Both large-Move rules test buildcfg.GORISCV64 before the side-effecting
logLargeCopy(v, s), so a single large copy is logged once by -json logopt
regardless of profile, instead of twice under rva23.
MemmoveKnownSize112-8 108.00n ± 0% 25.14n ± 0% -76.72% (p=0.000 n=10)
MemmoveKnownSize128-8 123.05n ± 0% 25.19n ± 0% -79.53% (p=0.000 n=10)
MemmoveKnownSize192-8 183.30n ± 0% 31.66n ± 0% -82.73% (p=0.000 n=10)
MemmoveKnownSize248-8 236.00n ± 0% 33.18n ± 0% -85.94% (p=0.000 n=10)
MemmoveKnownSize256-8 243.60n ± 0% 33.83n ± 0% -86.11% (p=0.000 n=10)
MemmoveKnownSize512-8 484.60n ± 0% 64.53n ± 0% -86.68% (p=0.000 n=10)
MemmoveKnownSize1024-8 972.6n ± 0% 125.6n ± 1% -87.09% (p=0.000 n=10)
CopyFat128-8 33.66n ± 1% 25.09n ± 0% -25.47% (p=0.000 n=10)
CopyFat256-8 65.31n ± 1% 33.15n ± 0% -49.24% (p=0.000 n=10)
CopyFat512-8 129.30n ± 0% 63.03n ± 0% -51.25% (p=0.000 n=10)
CopyFat520-8 130.50n ± 0% 82.79n ± 0% -36.56% (p=0.000 n=10)
CopyFat1024-8 257.3n ± 0% 123.3n ± 0% -52.08% (p=0.000 n=10)
CopyFat1032-8 257.4n ± 0% 143.7n ± 1% -44.19% (p=0.000 n=10)
CopyFat1040-8 258.6n ± 0% 139.9n ± 1% -45.91% (p=0.000 n=10)
MemmoveKnownSize112-8 989.4Mi ± 0% 4248.1Mi ± 0% +329.37% (p=0.000 n=10)
MemmoveKnownSize128-8 992.1Mi ± 0% 4846.3Mi ± 0% +388.48% (p=0.000 n=10)
MemmoveKnownSize192-8 999.1Mi ± 0% 5783.0Mi ± 0% +478.84% (p=0.000 n=10)
MemmoveKnownSize248-8 1002.1Mi ± 0% 7127.6Mi ± 0% +611.28% (p=0.000 n=10)
MemmoveKnownSize256-8 1002.4Mi ± 0% 7216.6Mi ± 0% +619.96% (p=0.000 n=10)
MemmoveKnownSize512-8 1007.6Mi ± 0% 7566.8Mi ± 0% +651.01% (p=0.000 n=10)
MemmoveKnownSize1024-8 1004.1Mi ± 0% 7775.2Mi ± 1% +674.32% (p=0.000 n=10)
Change-Id: I34368f52b599e1738abac9f13b9bed630af20db1
Change diff
diff --git a/src/cmd/compile/internal/riscv64/ssa.go b/src/cmd/compile/internal/riscv64/ssa.go
index f62eb14..047f650 100644
--- a/src/cmd/compile/internal/riscv64/ssa.go
+++ b/src/cmd/compile/internal/riscv64/ssa.go
@@ -927,6 +927,29 @@
if n <= 3*chunk {
v.Fatalf("MoveLoop too small:%d, expect:%d", n, 3*chunk)
}
+
+ // This op is emitted only for GORISCV64 < rva23 (see RISCV64.rules).
+ // Dispatch at runtime: use the RVV loop when the vector extension is
+ // usable (riscv64HasV), otherwise fall back to the scalar loop below.
+ // (For rva23+ the compiler emits LoweredMoveLoopV directly.)
+ flagM := s.Prog(riscv.AMOVBU)
+ flagM.From.Type = obj.TYPE_MEM
+ flagM.From.Name = obj.NAME_EXTERN
+ flagM.From.Sym = ir.Syms.RISCV64HasV
+ flagM.To.Type = obj.TYPE_REG
+ flagM.To.Reg = riscv.REG_X5
+
+ noVecM := s.Prog(riscv.ABEQ)
+ noVecM.From.Type = obj.TYPE_REG
+ noVecM.From.Reg = riscv.REG_X5
+ noVecM.Reg = riscv.REG_ZERO
+ noVecM.To.Type = obj.TYPE_BRANCH
+
+ moveLoopV(s, dst, src, n)
+
+ skipM := s.Prog(obj.AJMP)
+ skipM.To.Type = obj.TYPE_BRANCH
+
tmp := int16(riscv.REG_X5)
p := s.Prog(riscv.AADD)
@@ -935,6 +958,7 @@
p.Reg = src
p.To.Type = obj.TYPE_REG
p.To.Reg = riscv.REG_X6
+ noVecM.To.SetTarget(p) // scalar loop entry
for i := int64(0); i < 8; i++ {
moveOp(s, mov, dst, src, tmp, sz*i)
@@ -978,6 +1002,18 @@
n -= tsz
}
+ moveDoneV := s.Prog(obj.ANOP)
+ skipM.To.SetTarget(moveDoneV)
+
+ case ssa.OpRISCV64LoweredMoveLoopV:
+ dst := v.Args[0].Reg()
+ src := v.Args[1].Reg()
+ if dst == src {
+ break
+ }
+ n := v.AuxValAndOff().Val64()
+ moveLoopV(s, dst, src, n)
+
case ssa.OpRISCV64LoweredNilCheck:
// Issue a load which will fault if arg is nil.
p := s.Prog(riscv.AMOVB)
@@ -1200,6 +1236,88 @@
tail.To.SetTarget(done)
}
+func moveLoopV(s *ssagen.State, dst, src int16, n int64) {
+ if n == 0 {
+ return
+ }
+
+ const (
+ cntReg = riscv.REG_X5
+ vlReg = riscv.REG_X6
+ vecReg = riscv.REG_V24
+ )
+
+ p := s.Prog(riscv.AMOV)
+ p.From.Type = obj.TYPE_CONST
+ p.From.Offset = n
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = cntReg
+
+ vsetvli(s, cntReg, vlReg)
+
+ load := s.Prog(riscv.AVLE8V)
+ load.From.Type = obj.TYPE_MEM
+ load.From.Reg = src
+ load.To.Type = obj.TYPE_REG
+ load.To.Reg = vecReg
+
+ store := s.Prog(riscv.AVSE8V)
+ store.From.Type = obj.TYPE_REG
+ store.From.Reg = vecReg
+ store.To.Type = obj.TYPE_MEM
+ store.To.Reg = dst
+
+ p = s.Prog(riscv.AADD)
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = vlReg
+ p.Reg = src
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = src
+
+ p = s.Prog(riscv.AADD)
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = vlReg
+ p.Reg = dst
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = dst
+
+ p = s.Prog(riscv.ASUB)
+ p.From.Type = obj.TYPE_REG
+ p.From.Reg = vlReg
+ p.Reg = cntReg
+ p.To.Type = obj.TYPE_REG
+ p.To.Reg = cntReg
+
+ loop := s.Prog(riscv.ABGEU)
+ loop.From.Type = obj.TYPE_REG
+ loop.From.Reg = cntReg
+ loop.Reg = vlReg
+ loop.To.Type = obj.TYPE_BRANCH
+ loop.To.SetTarget(load)
+
+ tail := s.Prog(riscv.ABEQ)
+ tail.From.Type = obj.TYPE_REG
+ tail.From.Reg = cntReg
+ tail.Reg = riscv.REG_ZERO
+ tail.To.Type = obj.TYPE_BRANCH
+
+ vsetvli(s, cntReg, vlReg)
+ tailLoad := s.Prog(riscv.AVLE8V)
+ tailLoad.From.Type = obj.TYPE_MEM
+ tailLoad.From.Reg = src
+ tailLoad.To.Type = obj.TYPE_REG
+ tailLoad.To.Reg = vecReg
+
+ tailStore := s.Prog(riscv.AVSE8V)
+ tailStore.From.Type = obj.TYPE_REG
+ tailStore.From.Reg = vecReg
+ tailStore.To.Type = obj.TYPE_MEM
+ tailStore.To.Reg = dst
+
+ done := s.Prog(obj.ANOP)
+ tail.To.SetTarget(done)
+}
+
func vsetvli(s *ssagen.State, avlReg, vlReg int16) *obj.Prog {
p := s.Prog(riscv.AVSETVLI)
p.From.Type = obj.TYPE_REG
diff --git a/src/cmd/compile/internal/ssa/_gen/RISCV64.rules b/src/cmd/compile/internal/ssa/_gen/RISCV64.rules
index 6970492..495bceb 100644
--- a/src/cmd/compile/internal/ssa/_gen/RISCV64.rules
+++ b/src/cmd/compile/internal/ssa/_gen/RISCV64.rules
@@ -452,8 +452,11 @@
// Generic move uses a loop
(Move [s] {t} dst src mem) && s > 3*8*moveSize(t.Alignment(), config)
- && logLargeCopy(v, s) =>
+ && buildcfg.GORISCV64 < 23 && logLargeCopy(v, s) =>
(LoweredMoveLoop [makeValAndOff(int32(s),int32(t.Alignment()))] dst src mem)
+(Move [s] {t} dst src mem) && s > 3*8*moveSize(t.Alignment(), config)
+ && buildcfg.GORISCV64 >= 23 && logLargeCopy(v, s) =>
+ (LoweredMoveLoopV [makeValAndOff(int32(s),int32(t.Alignment()))] dst src mem)
// Boolean ops; 0=false, 1=true
(AndB ...) => (AND ...)
diff --git a/src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go b/src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go
index 72691e0..934ab37 100644
--- a/src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go
+++ b/src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go
@@ -401,6 +401,31 @@
faultOnNilArg1: true,
},
+ // general RVV unaligned move for rva23u64 and later
+ // arg0 = address of dst memory (clobber)
+ // arg1 = address of src memory (clobber)
+ // arg2 = mem
+ // auxint = size and type alignment
+ // returns mem
+ // Uses RVV V24-V31, which are not allocated by SSA. Async preemption
+ // saves and restores the RVV registers (see runtime.asyncPreempt and
+ // preempt_riscv64.go), so this is an ordinary safe point.
+ {
+ name: "LoweredMoveLoopV",
+ aux: "SymValAndOff",
+ typ: "Mem",
+ argLength: 3,
+ symEffect: "Write",
+ reg: regInfo{
+ inputs: []regMask{gpMask.minus(r5toR6), gpMask.minus(r5toR6)},
+ clobbers: r5toR6,
+ clobbersArg0: true,
+ clobbersArg1: true,
+ },
+ faultOnNilArg0: true,
+ faultOnNilArg1: true,
+ },
+
// Atomic loads.
// load from arg0. arg1=mem.
// returns <value,memory> so they can be properly ordered with other loads.
diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go
index eb160f2..851c132 100644
--- a/src/cmd/compile/internal/ssa/opGen.go
+++ b/src/cmd/compile/internal/ssa/opGen.go
@@ -6116,6 +6116,7 @@
OpRISCV64LoweredZeroLoopV
OpRISCV64LoweredMove
OpRISCV64LoweredMoveLoop
+ OpRISCV64LoweredMoveLoopV
OpRISCV64LoweredAtomicLoad8
OpRISCV64LoweredAtomicLoad32
OpRISCV64LoweredAtomicLoad64
@@ -95470,6 +95471,23 @@
},
},
{
+ name: "LoweredMoveLoopV",
+ auxType: auxSymValAndOff,
+ argLen: 3,
+ faultOnNilArg0: true,
+ faultOnNilArg1: true,
+ symEffect: SymWrite,
+ reg: regInfo{
+ inputs: []inputInfo{
+ {0, regMask{v1: 1006632896, v2: 0}}, // X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X28 X29 X30
+ {1, regMask{v1: 1006632896, v2: 0}}, // X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X28 X29 X30
+ },
+ clobbers: regMask{v1: 48, v2: 0}, // X5 X6
+ clobbersArg0: true,
+ clobbersArg1: true,
+ },
+ },
+ {
name: "LoweredAtomicLoad8",
argLen: 2,
faultOnNilArg0: true,
diff --git a/src/cmd/compile/internal/ssa/rewriteRISCV64.go b/src/cmd/compile/internal/ssa/rewriteRISCV64.go
index 6c8d11f..879d937 100644
--- a/src/cmd/compile/internal/ssa/rewriteRISCV64.go
+++ b/src/cmd/compile/internal/ssa/rewriteRISCV64.go
@@ -3118,7 +3118,7 @@
return true
}
// match: (Move [s] {t} dst src mem)
- // cond: s > 3*8*moveSize(t.Alignment(), config) && logLargeCopy(v, s)
+ // cond: s > 3*8*moveSize(t.Alignment(), config) && buildcfg.GORISCV64 < 23 && logLargeCopy(v, s)
// result: (LoweredMoveLoop [makeValAndOff(int32(s),int32(t.Alignment()))] dst src mem)
for {
s := auxIntToInt64(v.AuxInt)
@@ -3126,7 +3126,7 @@
dst := v_0
src := v_1
mem := v_2
- if !(s > 3*8*moveSize(t.Alignment(), config) && logLargeCopy(v, s)) {
+ if !(s > 3*8*moveSize(t.Alignment(), config) && buildcfg.GORISCV64 < 23 && logLargeCopy(v, s)) {
break
}
v.reset(OpRISCV64LoweredMoveLoop)
@@ -3134,6 +3134,23 @@
v.AddArg3(dst, src, mem)
return true
}
+ // match: (Move [s] {t} dst src mem)
+ // cond: s > 3*8*moveSize(t.Alignment(), config) && buildcfg.GORISCV64 >= 23 && logLargeCopy(v, s)
+ // result: (LoweredMoveLoopV [makeValAndOff(int32(s),int32(t.Alignment()))] dst src mem)
+ for {
+ s := auxIntToInt64(v.AuxInt)
+ t := auxToType(v.Aux)
+ dst := v_0
+ src := v_1
+ mem := v_2
+ if !(s > 3*8*moveSize(t.Alignment(), config) && buildcfg.GORISCV64 >= 23 && logLargeCopy(v, s)) {
+ break
+ }
+ v.reset(OpRISCV64LoweredMoveLoopV)
+ v.AuxInt = valAndOffToAuxInt(makeValAndOff(int32(s), int32(t.Alignment())))
+ v.AddArg3(dst, src, mem)
+ return true
+ }
return false
}
func rewriteValueRISCV64_OpNeq16(v *Value) bool {
Change information
Files:
- M src/cmd/compile/internal/riscv64/ssa.go
- M src/cmd/compile/internal/ssa/_gen/RISCV64.rules
- M src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go
- M src/cmd/compile/internal/ssa/opGen.go
- M src/cmd/compile/internal/ssa/rewriteRISCV64.go
Change size: M
Delta: 5 files changed, 184 insertions(+), 3 deletions(-)
Related details
Attention set is empty
Submit Requirements:
Code-Review
No-Unresolved-Comments
Review-Enforcement
TryBots-Pass
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Keith Randall (Gerrit)
Jul 21, 2026, 5:12:32 PM (4 days ago) Jul 21
to Julian Zhu, goph...@pubsubhelper.golang.org, Robert Griesemer, Keith Randall, Martin Möhrmann, Gopher Robot, golang-co...@googlegroups.com
Attention needed from Julian Zhu, Martin Möhrmann and Robert Griesemer
Keith Randall added 1 comment![]( "Open in Gerrit")
Patchset-level comments
File-level comment, Patchset 1 (Latest):
Keith Randall . unresolved
Can we get a riscv maintainer to take a look at these first?
Related details
Attention is currently required from:
- Julian Zhu
- Martin Möhrmann
- Robert Griesemer
Submit Requirements:
Code-Review
No-Unresolved-Comments
Review-Enforcement
TryBots-Pass
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Julian Zhu (Gerrit)
Jul 21, 2026, 10:51:42 PM (4 days ago) Jul 21
to goph...@pubsubhelper.golang.org, Joel Sing, Mark Ryan, Meng Zhuo, Robert Griesemer, Keith Randall, Martin Möhrmann, Gopher Robot, golang-co...@googlegroups.com
Attention needed from Keith Randall
Julian Zhu added 1 comment![]( "Open in Gerrit")
Patchset-level comments
Keith Randall . unresolved
Can we get a riscv maintainer to take a look at these first?
Julian Zhu
I'm in golang/riscv64 now. And this series CL depends on CL 801680 being approved by your team first.
Related details
Attention is currently required from:
- Keith Randall
Submit Requirements:
Code-Review
No-Unresolved-Comments
Review-Enforcement
TryBots-Pass
| Inspect html for hidden footers to help with email filtering. To unsubscribe visit settings. |
Reply all
Reply to author
Forward
0 new messages