diff --git a/src/cmd/compile/internal/ssagen/intrinsics.go b/src/cmd/compile/internal/ssagen/intrinsics.go
index f42d98d..770c98d 100644
--- a/src/cmd/compile/internal/ssagen/intrinsics.go
+++ b/src/cmd/compile/internal/ssagen/intrinsics.go
@@ -1942,7 +1942,7 @@
}
}
-func immJumpTable(s *state, idx *ssa.Value, intrinsicCall *ir.CallExpr, genOp func(*state, int)) *ssa.Value {
+func immJumpTable(s *state, idx *ssa.Value, intrinsicCall *ir.CallExpr, immMax int, genOp func(*state, int)) *ssa.Value {
// Make blocks we'll need.
bEnd := s.f.NewBlock(ssa.BlockPlain)
@@ -1950,20 +1950,49 @@
panic("immJumpTable expects uint8 value")
}
- // We will exhaust 0-255, so no need to check the bounds.
+ if immMax > 255 {
+ panic("immJumpTable expects immMax <= 255")
+ }
+
t := types.Types[types.TUINTPTR]
idx = s.conv(nil, idx, idx.Type, t)
- b := s.curBlock
- b.Kind = ssa.BlockJumpTable
- b.Pos = intrinsicCall.Pos()
+ var b *ssa.Block
+ // If immMax < 255, add bounds check before jump table.
+ if immMax < 255 {
+ // Check idx <= immMax
+ cmp := s.newValue2(s.ssaOp(ir.OLE, t), types.Types[types.TBOOL], idx, s.uintptrConstant(uint64(immMax)))
+ b = s.endBlock()
+ b.Kind = ssa.BlockIf
+ b.SetControl(cmp)
+ jt := s.f.NewBlock(ssa.BlockJumpTable)
+ panicBlock := s.f.NewBlock(ssa.BlockExit)
+ b.AddEdgeTo(jt)
+ b.AddEdgeTo(panicBlock)
+ b.Likely = ssa.BranchLikely
+
+ // Panic block for out-of-bounds index
+ s.startBlock(panicBlock)
+ s.rtcall(ir.Syms.PanicSimdImm, false, nil)
+ s.endBlock()
+
+ s.startBlock(jt)
+ b = s.curBlock
+ b.Pos = intrinsicCall.Pos()
+ } else {
+ // We will exhaust 0-255, so no need to check the bounds.
+ b = s.curBlock
+ b.Kind = ssa.BlockJumpTable
+ b.Pos = intrinsicCall.Pos()
+ }
+
if base.Flag.Cfg.SpectreIndex {
// Potential Spectre vulnerability hardening?
- idx = s.newValue2(ssa.OpSpectreSliceIndex, t, idx, s.uintptrConstant(255))
+ idx = s.newValue2(ssa.OpSpectreSliceIndex, t, idx, s.uintptrConstant(uint64(immMax)))
}
b.SetControl(idx)
targets := [256]*ssa.Block{}
- for i := range 256 {
+ for i := range immMax + 1 {
t := s.f.NewBlock(ssa.BlockPlain)
targets[i] = t
b.AddEdgeTo(t)
@@ -1971,11 +2000,12 @@
s.endBlock()
for i, t := range targets {
+ if i > immMax {
+ break
+ }
s.startBlock(t)
genOp(s, i)
- if t.Kind != ssa.BlockExit {
- t.AddEdgeTo(bEnd)
- }
+ t.AddEdgeTo(bEnd)
s.endBlock()
}
@@ -1984,48 +2014,60 @@
return ret
}
-func opLen1Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
+func opLen1Imm(op ssa.Op, t *types.Type, offset int, immMax int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
if args[1].Op == ssa.OpConst8 {
return s.newValue1I(op, t, args[1].AuxInt<<int64(offset), args[0])
}
- return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
+ return immJumpTable(s, args[1], n, immMax, func(sNew *state, idx int) {
// Encode as int8 due to requirement of AuxInt, check its comment for details.
s.vars[n] = sNew.newValue1I(op, t, int64(int8(idx<<offset)), args[0])
})
}
}
-func opLen2Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
+func opLen1Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
+ return opLen1Imm(op, t, offset, 255)
+}
+
+func opLen2Imm(op ssa.Op, t *types.Type, offset int, immMax int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
if args[1].Op == ssa.OpConst8 {
return s.newValue2I(op, t, args[1].AuxInt<<int64(offset), args[0], args[2])
}
- return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
+ return immJumpTable(s, args[1], n, immMax, func(sNew *state, idx int) {
// Encode as int8 due to requirement of AuxInt, check its comment for details.
s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx<<offset)), args[0], args[2])
})
}
}
-func opLen3Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
+func opLen2Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
+ return opLen2Imm(op, t, offset, 255)
+}
+
+func opLen3Imm(op ssa.Op, t *types.Type, offset int, immMax int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
if args[1].Op == ssa.OpConst8 {
return s.newValue3I(op, t, args[1].AuxInt<<int64(offset), args[0], args[2], args[3])
}
- return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
+ return immJumpTable(s, args[1], n, immMax, func(sNew *state, idx int) {
// Encode as int8 due to requirement of AuxInt, check its comment for details.
s.vars[n] = sNew.newValue3I(op, t, int64(int8(idx<<offset)), args[0], args[2], args[3])
})
}
}
+func opLen3Imm8(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
+ return opLen3Imm(op, t, offset, 255)
+}
+
func opLen2Imm8_2I(op ssa.Op, t *types.Type, offset int) func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
return func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value {
if args[2].Op == ssa.OpConst8 {
return s.newValue2I(op, t, args[2].AuxInt<<int64(offset), args[0], args[1])
}
- return immJumpTable(s, args[2], n, func(sNew *state, idx int) {
+ return immJumpTable(s, args[2], n, 255, func(sNew *state, idx int) {
// Encode as int8 due to requirement of AuxInt, check its comment for details.
s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx<<offset)), args[0], args[1])
})
@@ -2042,7 +2084,7 @@
four := s.constInt64(types.Types[types.TUINT8], 4)
shifted := s.newValue2(ssa.OpLsh8x8, types.Types[types.TUINT8], args[2], four)
combined := s.newValue2(ssa.OpAdd8, types.Types[types.TUINT8], args[1], shifted)
- return immJumpTable(s, combined, n, func(sNew *state, idx int) {
+ return immJumpTable(s, combined, n, 255, func(sNew *state, idx int) {
// Encode as int8 due to requirement of AuxInt, check its comment for details.
// TODO for "zeroing" values, panic instead.
if idx & ^(3+3<<4) == 0 {
@@ -2060,7 +2102,7 @@
if args[1].Op == ssa.OpConst8 {
return s.newValue2I(op, t, (args[1].AuxInt<<int64(offset))&0b11, args[0], args[2])
}
- return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
+ return immJumpTable(s, args[1], n, 255, func(sNew *state, idx int) {
// Encode as int8 due to requirement of AuxInt, check its comment for details.
s.vars[n] = sNew.newValue2I(op, t, int64(int8(idx<<offset))&0b11, args[0], args[2])
})
@@ -2072,7 +2114,7 @@
if args[2].Op == ssa.OpConst8 {
return s.newValue3I(op, t, args[2].AuxInt<<int64(offset), args[0], args[1], args[3])
}
- return immJumpTable(s, args[2], n, func(sNew *state, idx int) {
+ return immJumpTable(s, args[2], n, 255, func(sNew *state, idx int) {
// Encode as int8 due to requirement of AuxInt, check its comment for details.
s.vars[n] = sNew.newValue3I(op, t, int64(int8(idx<<offset)), args[0], args[1], args[3])
})
@@ -2084,7 +2126,7 @@
if args[1].Op == ssa.OpConst8 {
return s.newValue4I(op, t, args[1].AuxInt<<int64(offset), args[0], args[2], args[3], args[4])
}
- return immJumpTable(s, args[1], n, func(sNew *state, idx int) {
+ return immJumpTable(s, args[1], n, 255, func(sNew *state, idx int) {
// Encode as int8 due to requirement of AuxInt, check its comment for details.
s.vars[n] = sNew.newValue4I(op, t, int64(int8(idx<<offset)), args[0], args[2], args[3], args[4])
})
diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdGenericOps.go b/src/simd/archsimd/_gen/simdgen/gen_simdGenericOps.go
index 97a1b48..03cfecb 100644
--- a/src/simd/archsimd/_gen/simdgen/gen_simdGenericOps.go
+++ b/src/simd/archsimd/_gen/simdgen/gen_simdGenericOps.go
@@ -51,7 +51,7 @@
}
_, _, _, immType, gOp := op.shape()
gOpData := genericOpsData{gOp.GenericName(), len(gOp.In), op.Commutative}
- if immType == VarImm || immType == ConstVarImm {
+ if immType == VarImm || immType == VarImmLim || immType == ConstVarImm {
opsData.OpsImm = append(opsData.OpsImm, gOpData)
} else {
opsData.Ops = append(opsData.Ops, gOpData)
diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdIntrinsics.go b/src/simd/archsimd/_gen/simdgen/gen_simdIntrinsics.go
index 4e0ce87..d1a9f83 100644
--- a/src/simd/archsimd/_gen/simdgen/gen_simdIntrinsics.go
+++ b/src/simd/archsimd/_gen/simdgen/gen_simdIntrinsics.go
@@ -49,8 +49,12 @@
{{end}}
{{define "op4_31"}} addF(simdPackage, "{{(index .In 2).Go}}.{{.Go}}", opLen4_31(ssa.Op{{.GenericName}}, {{.SSAType}}), {{GetSysArch}})
{{end}}
+{{define "op1Imm"}} addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen1Imm(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}, {{(index .In 0).ImmMax}}), {{GetSysArch}})
+{{end}}
{{define "op1Imm8"}} addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen1Imm8(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})
{{end}}
+{{define "op2Imm"}} addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}, {{(index .In 0).ImmMax}}), {{GetSysArch}})
+{{end}}
{{define "op2Imm8"}} addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm8(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})
{{end}}
{{define "op2Imm8_2I"}} addF(simdPackage, "{{(index .In 1).Go}}.{{.Go}}", opLen2Imm8_2I(ssa.Op{{.GenericName}}, {{.SSAType}}, {{(index .In 0).ImmOffset}}), {{GetSysArch}})
diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go b/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go
index 30723a2..fa6cf3b 100644
--- a/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go
+++ b/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go
@@ -377,6 +377,8 @@
func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} uint8) {{.GoType}}
{{end}}
+{{define "op1Imm"}}{{template "op1Imm8" .}}{{end}}
+
{{define "op2Imm8"}}
{{if .Documentation}}{{.Documentation}}
//{{end}}
@@ -386,6 +388,8 @@
func ({{.Op1NameAndType "x"}}) {{.Go}}({{.ImmName}} uint8, {{.Op2NameAndType "y"}}) {{.GoType}}
{{end}}
+{{define "op2Imm"}}{{template "op2Imm8" .}}{{end}}
+
{{define "op2Imm8_2I"}}
{{if .Documentation}}{{.Documentation}}
//{{end}}
diff --git a/src/simd/archsimd/_gen/simdgen/gen_utility.go b/src/simd/archsimd/_gen/simdgen/gen_utility.go
index 60d5900..3972c84 100644
--- a/src/simd/archsimd/_gen/simdgen/gen_utility.go
+++ b/src/simd/archsimd/_gen/simdgen/gen_utility.go
@@ -112,6 +112,7 @@
ConstImm // const only immediate
VarImm // pure imm argument provided by the users
ConstVarImm // a combination of user arg and const
+ VarImmLim // pure imm argument provided by the users, up to maximum in op.ImmMax
)
const (
@@ -187,7 +188,11 @@
immType = ConstImm
}
} else if op.In[0].ImmOffset != nil {
- immType = VarImm
+ if op.In[0].ImmMax != nil {
+ immType = VarImmLim
+ } else {
+ immType = VarImm
+ }
} else {
panic(fmt.Errorf("simdgen requires imm to have at least one of ImmOffset or Const set: %s", op))
}
@@ -481,7 +486,7 @@
return op.In[4].OpNameAndType(s)
}
-var immClasses []string = []string{"BAD0Imm", "BAD1Imm", "op1Imm8", "op2Imm8", "op3Imm8", "op4Imm8"}
+var immClasses []string = []string{"BAD0Imm", "BAD1Imm", "op1Imm", "op2Imm", "op3Imm", "op4Imm"}
var classes []string = []string{"BAD0", "op1", "op2", "op3", "op4"}
// classifyOp returns a classification string, modified operation, and perhaps error based
@@ -495,12 +500,20 @@
var class string
- if immType == VarImm || immType == ConstVarImm {
+ if immType == VarImm || immType == VarImmLim || immType == ConstVarImm {
switch l := len(op.In); l {
case 1:
return "", op, fmt.Errorf("simdgen does not recognize this operation of only immediate input: %s", op)
case 2, 3, 4, 5:
- class = immClasses[l]
+ if immType == VarImmLim {
+ if len(op.In)-len(gOp.In) == 2 {
+ class = immClasses[l-1] // arm64: do not account const 0 imm in INS Vn[0], Vd[imm]
+ } else {
+ class = immClasses[l] // known immediate maximum value
+ }
+ } else {
+ class = immClasses[l] + "8" // default: imm up to 256
+ }
default:
return "", op, fmt.Errorf("simdgen does not recognize this operation of input length %d: %s", len(op.In), op)
}
diff --git a/src/simd/archsimd/_gen/simdgen/godefs.go b/src/simd/archsimd/_gen/simdgen/godefs.go
index 1f10442..670966a 100644
--- a/src/simd/archsimd/_gen/simdgen/godefs.go
+++ b/src/simd/archsimd/_gen/simdgen/godefs.go
@@ -302,6 +302,7 @@
// The compiler will right-shift the user-passed value by ImmOffset and set it as the AuxInt
// field of the operation.
ImmOffset *string
+ ImmMax *int // optional maximum immediate, also highest case in immediate jump table
Name *string // optional name in the Go intrinsic declaration
Lanes *int // *Lanes equals Bits/ElemBits except for scalars, when *Lanes == 1
// TreatLikeAScalarOfSize means only the lower $TreatLikeAScalarOfSize bits of the vector