[go] encoding/base64: Optimize DecodeString
Josselin Costanzi (Gerrit)
Josselin Costanzi has uploaded this change for review.
encoding/base64: Optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decode_strict method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/encoding/base64/base64.go
A src/encoding/base64/base64_generic32.go
A src/encoding/base64/base64_generic64.go
3 files changed, 232 insertions(+), 82 deletions(-)
diff --git a/src/encoding/base64/base64.go b/src/encoding/base64/base64.go
index d87d659..5f2a2e9 100644
--- a/src/encoding/base64/base64.go
+++ b/src/encoding/base64/base64.go
@@ -6,6 +6,7 @@
package base64
import (
+ "encoding/binary"
"io"
"strconv"
)
@@ -269,104 +270,101 @@
return "illegal base64 data at input byte " + strconv.FormatInt(int64(e), 10)
}
-// decode is like Decode but returns an additional 'end' value, which
-// indicates if end-of-message padding or a partial quantum was encountered
-// and thus any additional data is an error.
-func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) {
- si := 0
+// decode_quantum decode up to 4 base64 bytes. It returns the number of
+// bytes read from src, the number of bytes writen to dst, an 'end' value,
+// which indicates if end-of-message padding or a partial quantum was encountered
+// and thus any additional data is an error, and an error, if any.
+func (enc *Encoding) decode_quantum(dst, src []byte, si int) (nsi, n int, end bool, err error) {
+ // Decode quantum using the base64 alphabet
+ var dbuf [4]byte
+ dinc, dlen := 3, 4
- for si < len(src) && !end {
- // Decode quantum using the base64 alphabet
- var dbuf [4]byte
- dinc, dlen := 3, 4
-
- for j := 0; j < len(dbuf); j++ {
- if len(src) == si {
- if j == 0 {
- return n, false, nil
- } else if enc.padChar != NoPadding || j == 1 {
- return n, false, CorruptInputError(si - j)
- }
- dinc, dlen, end = j-1, j, true
- break
+ for j := 0; j < len(dbuf); j++ {
+ if len(src) == si {
+ if j == 0 {
+ return si, 0, true, nil
}
- in := src[si]
-
- si++
-
- dbuf[j] = enc.decodeMap[in]
- if dbuf[j] != 0xFF {
- continue
+ if enc.padChar != NoPadding || j == 1 {
+ return si, n, false, CorruptInputError(si - j)
}
- dbuf[j] = 0
+ dinc, dlen, end = j-1, j, true
+ break
+ }
+ in := src[si]
+ si++
- if in == '\n' || in == '\r' {
- j--
- continue
- }
- if rune(in) == enc.padChar {
- // We've reached the end and there's padding
- switch j {
- case 0, 1:
- // incorrect padding
- return n, false, CorruptInputError(si - 1)
- case 2:
- // "==" is expected, the first "=" is already consumed.
- // skip over newlines
- for si < len(src) && (src[si] == '\n' || src[si] == '\r') {
- si++
- }
- if si == len(src) {
- // not enough padding
- return n, false, CorruptInputError(len(src))
- }
- if rune(src[si]) != enc.padChar {
- // incorrect padding
- return n, false, CorruptInputError(si - 1)
- }
+ dbuf[j] = enc.decodeMap[in]
+ if dbuf[j] != 0xFF {
+ continue
+ }
+ dbuf[j] = 0
- si++
- }
+ if in == '\n' || in == '\r' {
+ j--
+ continue
+ }
+ if rune(in) == enc.padChar {
+ // We've reached the end and there's padding
+ switch j {
+ case 0, 1:
+ // incorrect padding
+ return si, n, false, CorruptInputError(si - 1)
+ case 2:
+ // "==" is expected, the first "=" is already consumed.
// skip over newlines
for si < len(src) && (src[si] == '\n' || src[si] == '\r') {
si++
}
- if si < len(src) {
- // trailing garbage
- err = CorruptInputError(si)
+ if si == len(src) {
+ // not enough padding
+ return si, n, false, CorruptInputError(len(src))
}
- dinc, dlen, end = 3, j, true
- break
- }
- return n, false, CorruptInputError(si - 1)
- }
+ if rune(src[si]) != enc.padChar {
+ // incorrect padding
+ return si, n, false, CorruptInputError(si - 1)
+ }
- // Convert 4x 6bit source bytes into 3 bytes
- val := uint(dbuf[0])<<18 | uint(dbuf[1])<<12 | uint(dbuf[2])<<6 | uint(dbuf[3])
- dbuf[2], dbuf[1], dbuf[0] = byte(val>>0), byte(val>>8), byte(val>>16)
- switch dlen {
- case 4:
- dst[2] = dbuf[2]
- dbuf[2] = 0
- fallthrough
- case 3:
- dst[1] = dbuf[1]
- if enc.strict && dbuf[2] != 0 {
- return n, end, CorruptInputError(si - 1)
+ si++
}
- dbuf[1] = 0
- fallthrough
- case 2:
- dst[0] = dbuf[0]
- if enc.strict && (dbuf[1] != 0 || dbuf[2] != 0) {
- return n, end, CorruptInputError(si - 2)
+ // skip over newlines
+ for si < len(src) && (src[si] == '\n' || src[si] == '\r') {
+ si++
}
+ if si < len(src) {
+ // trailing garbage
+ err = CorruptInputError(si)
+ }
+ dinc, dlen, end = 3, j, true
+ break
}
- dst = dst[dinc:]
- n += dlen - 1
+ return si, n, false, CorruptInputError(si - 1)
}
- return n, end, err
+ // Convert 4x 6bit source bytes into 3 bytes
+ val := uint(dbuf[0])<<18 | uint(dbuf[1])<<12 | uint(dbuf[2])<<6 | uint(dbuf[3])
+ dbuf[2], dbuf[1], dbuf[0] = byte(val>>0), byte(val>>8), byte(val>>16)
+ switch dlen {
+ case 4:
+ dst[2] = dbuf[2]
+ dbuf[2] = 0
+ fallthrough
+ case 3:
+ dst[1] = dbuf[1]
+ if enc.strict && dbuf[2] != 0 {
+ return si, n, end, CorruptInputError(si - 1)
+ }
+ dbuf[1] = 0
+ fallthrough
+ case 2:
+ dst[0] = dbuf[0]
+ if enc.strict && (dbuf[1] != 0 || dbuf[2] != 0) {
+ return si, n, end, CorruptInputError(si - 2)
+ }
+ }
+ dst = dst[dinc:]
+ n += dlen - 1
+
+ return si, n, end, err
}
// Decode decodes src using the encoding enc. It writes at most
@@ -465,6 +463,82 @@
return n, d.err
}
+// decode32 tries to decode 4 base64 char into 3 bytes.
+// len(dst) and len(src) must both be >= 4.
+// Returns true if decode succeeded.
+func (enc *Encoding) decode32(dst, src []byte) bool {
+ var str, res uint32
+ var dec uint32
+
+ str = binary.BigEndian.Uint32(src)
+
+ if dec = uint32(enc.decodeMap[(str>>24)&0xff]); dec > 63 {
+ return false
+ }
+ res = dec << 26
+ if dec = uint32(enc.decodeMap[(str>>16)&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 20
+ if dec = uint32(enc.decodeMap[(str>>8)&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 14
+ if dec = uint32(enc.decodeMap[str&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 8
+
+ binary.BigEndian.PutUint32(dst, res)
+ return true
+}
+
+// decode64 tries to decode 8 base64 char into 6 bytes.
+// len(dst) and len(src) must both be >= 8.
+// Returns true if decode succeeded.
+func (enc *Encoding) decode64(dst, src []byte) bool {
+ var str, res uint64
+ var dec uint64
+
+ str = binary.BigEndian.Uint64(src)
+ if dec = uint64(enc.decodeMap[(str>>56)&0xff]); dec > 63 {
+ return false
+ }
+ res = dec << 58
+ if dec = uint64(enc.decodeMap[(str>>48)&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 52
+ if dec = uint64(enc.decodeMap[(str>>40)&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 46
+ if dec = uint64(enc.decodeMap[(str>>32)&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 40
+ if dec = uint64(enc.decodeMap[(str>>24)&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 34
+ if dec = uint64(enc.decodeMap[(str>>16)&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 28
+ if dec = uint64(enc.decodeMap[(str>>8)&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 22
+ if dec = uint64(enc.decodeMap[str&0xff]); dec > 63 {
+ return false
+ }
+ res |= dec << 16
+
+ binary.BigEndian.PutUint64(dst, res)
+ return true
+
+}
+
type newlineFilteringReader struct {
wrapped io.Reader
}
diff --git a/src/encoding/base64/base64_generic32.go b/src/encoding/base64/base64_generic32.go
new file mode 100644
index 0000000..0e692af
--- /dev/null
+++ b/src/encoding/base64/base64_generic32.go
@@ -0,0 +1,34 @@
+// +build 386 arm armbe mips mipsle ppc s390 sparc
+
+package base64
+
+func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) {
+ var ninc, si, ns int
+
+ if len(src) == 0 {
+ return 0, true, nil
+ }
+
+ ilen := len(src)
+ olen := len(dst)
+ for ilen-si >= 4 && olen-n >= 4 {
+ if ok := enc.decode32(dst[n:], src[si:]); ok {
+ n += 3
+ si += 4
+ continue
+ }
+ si, ninc, end, err = enc.decode_quantum(dst[n:], src, si)
+ if err != nil {
+ return n + ninc, end, err
+ }
+ n += ninc
+ }
+ for si < len(src) {
+ si, ns, end, err = enc.decode_quantum(dst[n:], src, si)
+ n += ns
+ if end || err != nil {
+ return n, end, err
+ }
+ }
+ return n, end, err
+}
diff --git a/src/encoding/base64/base64_generic64.go b/src/encoding/base64/base64_generic64.go
new file mode 100644
index 0000000..2d68527
--- /dev/null
+++ b/src/encoding/base64/base64_generic64.go
@@ -0,0 +1,42 @@
+// +build amd64 amd64p32 arm64 arm64be ppc64 ppc64le mips64 mips64le mips64p32 mips64p32le s390x sparc64
+
+package base64
+
+func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) {
+ var ninc, si int
+
+ if len(src) == 0 {
+ return 0, true, nil
+ }
+
+ ilen := len(src)
+ olen := len(dst)
+ for ilen-si >= 8 && olen-n >= 8 {
+ if ok := enc.decode64(dst[n:], src[si:]); ok {
+ n += 6
+ si += 8
+ continue
+ }
+ si, ninc, end, err = enc.decode_quantum(dst[n:], src, si)
+ if err != nil {
+ return n + ninc, end, err
+ }
+ n += ninc
+ }
+
+ if ilen-si >= 4 && olen-n >= 4 {
+ if ok := enc.decode32(dst[n:], src[si:]); ok {
+ n += 3
+ si += 4
+ }
+ }
+
+ for si < len(src) {
+ si, ninc, end, err = enc.decode_quantum(dst[n:], src, si)
+ n += ninc
+ if end || err != nil {
+ return n, end, err
+ }
+ }
+ return n, end, err
+}
To view, visit change 38632. To unsubscribe, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi posted comments on this change.
Patch set 1:
This CL comes after https://go-review.googlesource.com/#/c/34950/
Josselin Costanzi (Gerrit)
(1 comment)
Patch Set #1, Line 11:
complex decode_strict method when a non-base64 character is present.*decode_quantum
To view, visit change 38632. To unsubscribe, visit settings.
Brad Fitzpatrick (Gerrit)
Brad Fitzpatrick posted comments on this change.
Patch set 1:
Patch Set 1:
This CL comes after https://go-review.googlesource.com/#/c/34950/
Why? Why not one CL?
It's hard to review stuff if you keep updating the CLs and stacking more on top.
Let's do one thing at a time and stop updating things.
Josselin Costanzi (Gerrit)
Josselin Costanzi posted comments on this change.
Patch set 1:
Patch Set 1:
> This CL comes after https://go-review.googlesource.com/#/c/34950/Why? Why not one CL?
It's hard to review stuff if you keep updating the CLs and stacking
more on top.Let's do one thing at a time and stop updating things.
It could be in the same CL but I'm not sure how to do this (I hoped it would be automatic as the two commits were following each other in my local branch).
I'm sorry for constantly updating CL 34950, I won't touch it unless asked to.
Josselin Costanzi (Gerrit)
Josselin Costanzi uploaded patch set #2 to this change.
encoding/base64: Optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decode_quantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/encoding/base64/base64.go
A src/encoding/base64/base64_generic32.go
A src/encoding/base64/base64_generic64.go
3 files changed, 233 insertions(+), 84 deletions(-)
To view, visit change 38632. To unsubscribe, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi posted comments on this change.
Patch set 2:
Rebased on top of current master
Brad Fitzpatrick (Gerrit)
Brad Fitzpatrick posted comments on this change.
Patch set 2:
Missing copyright headers.
But I think you can actually do this all with Go consts in one file (and no build tags) and get the same generated code everywhere.
Josselin Costanzi (Gerrit)
Josselin Costanzi uploaded patch set #3 to this change.
encoding/base64: Optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decode_quantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/encoding/base64/base64.go
1 file changed, 201 insertions(+), 84 deletions(-)
To view, visit change 38632. To unsubscribe, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi posted comments on this change.
Patch set 3:
Patch Set 2:
Missing copyright headers.
But I think you can actually do this all with Go consts in one file (and no build tags) and get the same generated code
thanks, great idea!
(1 comment)
File src/encoding/base64/base64.go:
Patch Set #3, Line 274:
// bytes read from src, the number of bytes writen to dst, an 'end' value,I should re-phrase this, what about:
decode_quantum decode up to 4 base64 bytes. For parameters, it takes a destination and source buffer and si, the index at which we read the source buffer.
It returns an updated source index nsi, n the number of bytes writen to dst, an 'end' value which indicates if end-of-messsage padding or a partial quantum was encountered and thus any additional data is an error, and an error, if any.
To view, visit change 38632. To unsubscribe, visit settings.
Aliaksandr Valialkin (Gerrit)
Aliaksandr Valialkin posted comments on this change.
Patch set 3:
(16 comments)
File src/encoding/base64/base64.go:
Patch Set #3, Line 9:
"encoding/binary"Update cmd/dist/deps.go with cmd/dist/mkdeps.bash
Patch Set #3, Line 273:
decodes/decode/decodes/
Patch Set #3, Line 274:
// bytes read from src, the number of bytes writen to dst, an 'end' value,I should re-phrase this, what about:
IHMO the existing comment is better than the re-phrased one :)
Patch Set #3, Line 277:
decode_quantums/decode_quantum/decodeQuantum/
n should be substituted by 0, since we didn't write anything to dst yet
s/0xFF/0xff/
to be consistent with 0xff lines in decode32 and decode64 below
substitute n with 0
s/n/0/
s/n/0/
s/n/0/
Patch Set #3, Line 340:
return si, n, false, CorruptInputError(si - 1)It would be better to move this line before the 'if' condition above in order to reduce code indentation:
if rune(in) != enc.padChar {
return si, 0, false, CorruptInputError(si - 1)
}
// We've reached the end and there's padding
...
Missing continue. It should trigger if strconv.IntSize < 64
var str, res uint32
var dec uint32Rename:
str -> sn ("source number")
res -> dn ("destination number")
dec -> n ("number")The same is for the decode64
Patch Set #3, Line 517:
str = binary.BigEndian.Uint32(src)Define 'sn' on this line:
sn := binary.BigEndian.Uint32(src)
Patch Set #3, Line 522:
res = dec << 26Write this line like:
dn |= n << 26
In order to be consistent with the similar lines below
(sn >> 0)&0xff
to be consistent with the lines above
To view, visit change 38632. To unsubscribe, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi posted comments on this change.
Patch set 3:
Thanks Aliaksandr.
(16 comments)
Patch Set #3, Line 9:
"encoding/binary"Update cmd/dist/deps.go with cmd/dist/mkdeps.bash
Done
Patch Set #3, Line 273:
decodes/decode/decodes/
Done
Patch Set #3, Line 274:
// bytes read from src, the number of bytes writen to dst, an 'end' value,
IHMO the existing comment is better than the re-phrased one :)
Ack
Patch Set #3, Line 277:
decode_quantums/decode_quantum/decodeQuantum/
Done
n should be substituted by 0, since we didn't write anything to dst yet
Done
s/0xFF/0xff/
Done
substitute n with 0
Done
s/n/0/
Done
s/n/0/
Done
Patch Set #3, Line 340:
return si, n, false, CorruptInputError(si - 1)It would be better to move this line before the 'if' condition above in ord
Done
s/n/0/
Done
Missing continue. It should trigger if strconv.IntSize < 64
Ack
Good catch, thanks
var str, res uint32
var dec uint32Rename:
Done
Patch Set #3, Line 517:
str = binary.BigEndian.Uint32(src)Define 'sn' on this line:
Done
Patch Set #3, Line 522:
res = dec << 26Write this line like:
Done
(sn >> 0)&0xff
Done
To view, visit change 38632. To unsubscribe, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi uploaded patch set #4 to this change.
encoding/base64: Optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decode_quantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/cmd/dist/deps.go
M src/encoding/base64/base64.go
M src/go/build/deps_test.go
3 files changed, 214 insertions(+), 95 deletions(-)
To view, visit change 38632. To unsubscribe, visit settings.
Aliaksandr Valialkin (Gerrit)
Aliaksandr Valialkin posted comments on this change.
Patch set 4:Code-Review +1
LGTM
Aliaksandr Valialkin (Gerrit)
(1 comment)
Patch Set #4, Line 11:
decode_quantums/decode_quantum/decodeQuantum/
To view, visit change 38632. To unsubscribe, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi uploaded patch set #5 to this change.
encoding/base64: Optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decodeQuantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/cmd/dist/deps.go
M src/encoding/base64/base64.go
M src/go/build/deps_test.go
3 files changed, 214 insertions(+), 95 deletions(-)
To view, visit change 38632. To unsubscribe, visit settings.
Brad Fitzpatrick (Gerrit)
Brad Fitzpatrick posted comments on this change.
Patch set 5:
(1 comment)
File src/encoding/base64/base64.go:
Patch Set #5, Line 277:
si intyou don't document this 'si' parameter.
But if it's just an index into 'src', I think you should remove it and just have the caller pass in a re-sliced src.
To view, visit change 38632. To unsubscribe, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi posted comments on this change.
Patch set 5:
Patch Set 5:
(1 comment)
si is the index for src. I need it in decodeQuantum so the error index are correct. Maybe there is another way to do this?
I'm away from my computer for the next week and if I'm correct, we are reaching the end of the merge period for new code so barring any other comment, if we decide it's a matter of documenting si, we can merge this patch and I'll improve the comment in another patch when I'm back. Otherwise we shouldn't rush it and this can wait until it's up to our standards of quality.
Josselin Costanzi (Gerrit)
Josselin Costanzi uploaded patch set #6 to this change.
encoding/base64: Optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decodeQuantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/cmd/dist/deps.go
M src/encoding/base64/base64.go
M src/go/build/deps_test.go
3 files changed, 217 insertions(+), 95 deletions(-)
To view, visit change 38632. To unsubscribe, visit settings.
Brad Fitzpatrick (Gerrit)
Brad Fitzpatrick posted comments on this change.
Patch set 6:
R=go1.10
Aliaksandr Valialkin (Gerrit)
Josselin, could you rebase this patch on master branch, so it could be merged into go1.10?
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Josselin Costanzi (Gerrit)
Patch Set 6:
Josselin, could you rebase this patch on master branch, so it could be merged into go1.10?
Sure, I'll try to do that this weekend!
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi uploaded patch set #7 to this change.
encoding/base64: Optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decodeQuantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/cmd/dist/deps.go
M src/encoding/base64/base64.go
M src/go/build/deps_test.go
3 files changed, 214 insertions(+), 93 deletions(-)
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Ian Lance Taylor (Gerrit)
6 comments:
Patch Set #7, Line 7:
encoding/base64: Optimize DecodeStrings/Optimize/optimize/
File src/encoding/base64/base64.go:
Patch Set #7, Line 472:
var ninc, si intDon't define these up here--this isn't C. At least for si the initial vale is important, so define it below, near ilen and olen, using
si := 0Patch Set #7, Line 486:
si, ninc, end, err = enc.decodeQuantum(dst[n:], src, si)Calling decodeQuantum each time around the loop seems pointless. Seems like you should only call it when decoede64 encounters a character it doesn't recognize.
Patch Set #7, Line 522:
sn := binary.BigEndian.Uint32(src)Calling Uint32 here does not seem like a win. That will read four bytes and turn them into a 32-bit integer, which you then pull apart below. Why not simply refers to src[0], src[1], src[2], src[3]?
Patch Set #7, Line 524:
n > 63Instead of writing `n > 63` seems like you should write `n == 0xff`, since that is the signal that there is no output for a given input.
Patch Set #7, Line 541:
binary.BigEndian.PutUint32(dst, dn)Similarly, why not set dst[0], etc.
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Josselin Costanzi (Gerrit)
5 comments:
Patch Set #7, Line 472:
var ninc, si int
Don't define these up here--this isn't C. […]
Done
Patch Set #7, Line 486:
si, ninc, end, err = enc.decodeQuantum(dst[n:], src, si)
Calling decodeQuantum each time around the loop seems pointless. […]
decodeQuantum isn't called every loop thanks to the continue, just above.
Maybe a clearer construct is:
for ... {
if ok := enc.decode64(dst[n:], src[si:]); ok {
...
} else {
enc.decodeQuantum(dst[n:], src, si)
...
}
}
Patch Set #7, Line 522:
sn := binary.BigEndian.Uint32(src)
Calling Uint32 here does not seem like a win. […]
Indeed, I took my inspiration from aklomp/base64 too literally.
Patch Set #7, Line 524:
n > 63
Instead of writing `n > 63` seems like you should write `n == 0xff`, since that is the signal that t […]
Done
Patch Set #7, Line 541:
binary.BigEndian.PutUint32(dst, dn)Similarly, why not set dst[0], etc.
That one will be more complicated as you go from an uint32 to bytes. binary.BigEndian.PutUint32 seems to be the correct and efficient function to do this.
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi uploaded patch set #8 to this change.
encoding/base64: optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decodeQuantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/cmd/dist/deps.go
M src/encoding/base64/base64.go
M src/go/build/deps_test.go
3 files changed, 214 insertions(+), 95 deletions(-)
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Ian Lance Taylor (Gerrit)
Thanks.
6 comments:
This change and the one for path/filepath seems unrelated. Not sure what caused that.
File src/encoding/base64/base64.go:
Patch Set #8, Line 276:
// It returns the number of bytes read from src, the number of bytes writens/writen/written/
Patch Set #8, Line 289:
return si, 0, true, nilThe old code returns end == false in this case. I can't figure out why it changed. Was it wrong before?
Patch Set #8, Line 471:
func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) {This method should have a comment explaining what it does, perhaps simply the doc comment it used to have.
Patch Set #8, Line 472:
var ninc intIt looks like ninc doesn't need to be defined across the whole function. You should add `var ninc int` before each call to decodeQuantum.
Remove this blank line.
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Josselin Costanzi (Gerrit)
1 comment:
File src/encoding/base64/base64.go:
Patch Set #8, Line 289:
return si, 0, true, nilThe old code returns end == false in this case. I can't figure out why it changed. […]
This must be a mistake on my part.
After looking at the code and at the reason why changing "false" to "true" didn't raise any error in the unit tests, it seems this 'end' value isn't used anywhere. I'll remove it.
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Josselin Costanzi (Gerrit)
Josselin Costanzi uploaded patch set #9 to this change.
encoding/base64: optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decodeQuantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
---
M src/cmd/dist/deps.go
M src/encoding/base64/base64.go
M src/encoding/base64/base64_test.go
M src/go/build/deps_test.go
4 files changed, 218 insertions(+), 112 deletions(-)
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.
Ian Lance Taylor (Gerrit)
Patch set 9:Run-TryBot +1
Gobot Gobot (Gerrit)
TryBots beginning. Status page: https://farmer.golang.org/try?commit=b5ac0171
Gobot Gobot (Gerrit)
TryBots are happy.
Patch set 9:TryBot-Result +1
Ian Lance Taylor (Gerrit)
Thanks!
Patch set 9:Code-Review +2
Ian Lance Taylor (Gerrit)
Ian Lance Taylor merged this change.
encoding/base64: optimize DecodeString
Optimize base64 decoding speed by adding 32-bits and 64-bits specialized
methods that don't perform any error checking and fall back to the more
complex decodeQuantum method when a non-base64 character is present.
On a 64-bits cpu:
name old time/op new time/op delta
DecodeString/2-4 70.0ns ± 6% 69.2ns ± 0% ~ (p=0.169 n=5+8)
DecodeString/4-4 91.3ns ± 2% 80.4ns ± 0% -11.89% (p=0.001 n=5+10)
DecodeString/8-4 126ns ± 5% 106ns ± 0% -16.14% (p=0.000 n=5+7)
DecodeString/64-4 652ns ±21% 361ns ± 0% -44.57% (p=0.000 n=5+7)
DecodeString/8192-4 61.0µs ±13% 31.5µs ± 1% -48.38% (p=0.001 n=5+9)
name old speed new speed delta
DecodeString/2-4 57.2MB/s ± 6% 57.7MB/s ± 2% ~ (p=0.419 n=5+9)
DecodeString/4-4 87.7MB/s ± 2% 99.5MB/s ± 0% +13.45% (p=0.001 n=5+10)
DecodeString/8-4 94.8MB/s ± 5% 112.6MB/s ± 1% +18.82% (p=0.001 n=5+9)
DecodeString/64-4 136MB/s ±19% 243MB/s ± 0% +78.17% (p=0.003 n=5+7)
DecodeString/8192-4 180MB/s ±11% 347MB/s ± 1% +92.94% (p=0.001 n=5+9)
Improves #19636
Change-Id: Ic10a454851093a7e1d46ca0c140deed73535d990
Reviewed-on: https://go-review.googlesource.com/38632
Run-TryBot: Ian Lance Taylor <ia...@golang.org>
TryBot-Result: Gobot Gobot <go...@golang.org>
Reviewed-by: Ian Lance Taylor <ia...@golang.org>
---
M src/cmd/dist/deps.go
M src/encoding/base64/base64.go
M src/encoding/base64/base64_test.go
M src/go/build/deps_test.go
4 files changed, 218 insertions(+), 112 deletions(-)
diff --git a/src/cmd/dist/deps.go b/src/cmd/dist/deps.go
index cd7eaae..660db75 100644
--- a/src/cmd/dist/deps.go
+++ b/src/cmd/dist/deps.go
@@ -461,8 +461,9 @@
},
"encoding/base64": {
- "io", // encoding/base64
- "strconv", // encoding/base64
+ "encoding/binary", // encoding/base64
+ "io", // encoding/base64
+ "strconv", // encoding/base64
},
"encoding/binary": {
diff --git a/src/encoding/base64/base64.go b/src/encoding/base64/base64.go
index b208f9e..9a99370 100644
--- a/src/encoding/base64/base64.go
+++ b/src/encoding/base64/base64.go
@@ -6,6 +6,7 @@
package base64
import (
+ "encoding/binary"
"io"
"strconv"
)
@@ -269,121 +270,110 @@
return "illegal base64 data at input byte " + strconv.FormatInt(int64(e), 10)
}
-// decode is like Decode but returns an additional 'end' value, which
-// indicates if end-of-message padding or a partial quantum was encountered
-// and thus any additional data is an error.
-func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) {
- si := 0
+// decodeQuantum decodes up to 4 base64 bytes. It takes for parameters
+// the destination buffer dst, the source buffer src and an index in the
+// source buffer si.
+// It returns the number of bytes read from src, the number of bytes written
+// to dst, and an error, if any.
+func (enc *Encoding) decodeQuantum(dst, src []byte, si int) (nsi, n int, err error) {
+ // Decode quantum using the base64 alphabet
+ var dbuf [4]byte
+ dinc, dlen := 3, 4
- for si < len(src) && !end {
- // Decode quantum using the base64 alphabet
- var dbuf [4]byte
- dinc, dlen := 3, 4
-
- for j := 0; j < len(dbuf); j++ {
- if len(src) == si {
- switch {
- case j == 0:
- return n, false, nil
- case j == 1, enc.padChar != NoPadding:
- return n, false, CorruptInputError(si - j)
- }
- dinc, dlen, end = j-1, j, true
- break
+ for j := 0; j < len(dbuf); j++ {
+ if len(src) == si {
+ switch {
+ case j == 0:
+ return si, 0, nil
+ case j == 1, enc.padChar != NoPadding:
+ return si, 0, CorruptInputError(si - j)
}
- in := src[si]
+ dinc, dlen = j-1, j
+ break
+ }
+ in := src[si]
+ si++
+
+ out := enc.decodeMap[in]
+ if out != 0xff {
+ dbuf[j] = out
+ continue
+ }
+
+ if in == '\n' || in == '\r' {
+ j--
+ continue
+ }
+
+ if rune(in) != enc.padChar {
+ return si, 0, CorruptInputError(si - 1)
+ }
+
+ // We've reached the end and there's padding
+ switch j {
+ case 0, 1:
+ // incorrect padding
+ return si, 0, CorruptInputError(si - 1)
+ case 2:
+ // "==" is expected, the first "=" is already consumed.
+ // skip over newlines
+ for si < len(src) && (src[si] == '\n' || src[si] == '\r') {
+ si++
+ }
+ if si == len(src) {
+ // not enough padding
+ return si, 0, CorruptInputError(len(src))
+ }
+ if rune(src[si]) != enc.padChar {
+ // incorrect padding
+ return si, 0, CorruptInputError(si - 1)
+ }
si++
-
- out := enc.decodeMap[in]
- if out != 0xFF {
- dbuf[j] = out
- continue
- }
-
- if in == '\n' || in == '\r' {
- j--
- continue
- }
- if rune(in) == enc.padChar {
- // We've reached the end and there's padding
- switch j {
- case 0, 1:
- // incorrect padding
- return n, false, CorruptInputError(si - 1)
- case 2:
- // "==" is expected, the first "=" is already consumed.
- // skip over newlines
- for si < len(src) && (src[si] == '\n' || src[si] == '\r') {
- si++
- }
- if si == len(src) {
- // not enough padding
- return n, false, CorruptInputError(len(src))
- }
- if rune(src[si]) != enc.padChar {
- // incorrect padding
- return n, false, CorruptInputError(si - 1)
- }
-
- si++
- }
- // skip over newlines
- for si < len(src) && (src[si] == '\n' || src[si] == '\r') {
- si++
- }
- if si < len(src) {
- // trailing garbage
- err = CorruptInputError(si)
- }
- dinc, dlen, end = 3, j, true
- break
- }
- return n, false, CorruptInputError(si - 1)
}
- // Convert 4x 6bit source bytes into 3 bytes
- val := uint(dbuf[0])<<18 | uint(dbuf[1])<<12 | uint(dbuf[2])<<6 | uint(dbuf[3])
- dbuf[2], dbuf[1], dbuf[0] = byte(val>>0), byte(val>>8), byte(val>>16)
- switch dlen {
- case 4:
- dst[2] = dbuf[2]
- dbuf[2] = 0
- fallthrough
- case 3:
- dst[1] = dbuf[1]
- if enc.strict && dbuf[2] != 0 {
- return n, end, CorruptInputError(si - 1)
- }
- dbuf[1] = 0
- fallthrough
- case 2:
- dst[0] = dbuf[0]
- if enc.strict && (dbuf[1] != 0 || dbuf[2] != 0) {
- return n, end, CorruptInputError(si - 2)
- }
+ // skip over newlines
+ for si < len(src) && (src[si] == '\n' || src[si] == '\r') {
+ si++
}
- dst = dst[dinc:]
- n += dlen - 1
+ if si < len(src) {
+ // trailing garbage
+ err = CorruptInputError(si)
+ }
+ dinc, dlen = 3, j
+ break
}
- return n, end, err
-}
+ // Convert 4x 6bit source bytes into 3 bytes
+ val := uint(dbuf[0])<<18 | uint(dbuf[1])<<12 | uint(dbuf[2])<<6 | uint(dbuf[3])
+ dbuf[2], dbuf[1], dbuf[0] = byte(val>>0), byte(val>>8), byte(val>>16)
+ switch dlen {
+ case 4:
+ dst[2] = dbuf[2]
+ dbuf[2] = 0
+ fallthrough
+ case 3:
+ dst[1] = dbuf[1]
+ if enc.strict && dbuf[2] != 0 {
+ return si, 0, CorruptInputError(si - 1)
+ }
+ dbuf[1] = 0
+ fallthrough
+ case 2:
+ dst[0] = dbuf[0]
+ if enc.strict && (dbuf[1] != 0 || dbuf[2] != 0) {
+ return si, 0, CorruptInputError(si - 2)
+ }
+ }
+ dst = dst[dinc:]
-// Decode decodes src using the encoding enc. It writes at most
-// DecodedLen(len(src)) bytes to dst and returns the number of bytes
-// written. If src contains invalid base64 data, it will return the
-// number of bytes successfully written and CorruptInputError.
-// New line characters (\r and \n) are ignored.
-func (enc *Encoding) Decode(dst, src []byte) (n int, err error) {
- n, _, err = enc.decode(dst, src)
- return
+ return si, dlen - 1, err
}
// DecodeString returns the bytes represented by the base64 string s.
func (enc *Encoding) DecodeString(s string) ([]byte, error) {
dbuf := make([]byte, enc.DecodedLen(len(s)))
- n, _, err := enc.decode(dbuf, []byte(s))
+ n, err := enc.Decode(dbuf, []byte(s))
return dbuf[:n], err
}
@@ -392,7 +382,6 @@
readErr error // error from r.Read
enc *Encoding
r io.Reader
- end bool // saw end of message
buf [1024]byte // leftover input
nbuf int
out []byte // leftover decoded output
@@ -430,9 +419,8 @@
if d.enc.padChar == NoPadding && d.nbuf > 0 {
// Decode final fragment, without padding.
var nw int
- nw, _, d.err = d.enc.decode(d.outbuf[:], d.buf[:d.nbuf])
+ nw, d.err = d.enc.Decode(d.outbuf[:], d.buf[:d.nbuf])
d.nbuf = 0
- d.end = true
d.out = d.outbuf[:nw]
n = copy(p, d.out)
d.out = d.out[n:]
@@ -454,18 +442,138 @@
nr := d.nbuf / 4 * 4
nw := d.nbuf / 4 * 3
if nw > len(p) {
- nw, d.end, d.err = d.enc.decode(d.outbuf[:], d.buf[:nr])
+ nw, d.err = d.enc.Decode(d.outbuf[:], d.buf[:nr])
d.out = d.outbuf[:nw]
n = copy(p, d.out)
d.out = d.out[n:]
} else {
- n, d.end, d.err = d.enc.decode(p, d.buf[:nr])
+ n, d.err = d.enc.Decode(p, d.buf[:nr])
}
d.nbuf -= nr
copy(d.buf[:d.nbuf], d.buf[nr:])
return n, d.err
}
+// Decode decodes src using the encoding enc. It writes at most
+// DecodedLen(len(src)) bytes to dst and returns the number of bytes
+// written. If src contains invalid base64 data, it will return the
+// number of bytes successfully written and CorruptInputError.
+// New line characters (\r and \n) are ignored.
+func (enc *Encoding) Decode(dst, src []byte) (n int, err error) {
+ if len(src) == 0 {
+ return 0, nil
+ }
+
+ si := 0
+ ilen := len(src)
+ olen := len(dst)
+ for strconv.IntSize >= 64 && ilen-si >= 8 && olen-n >= 8 {
+ if ok := enc.decode64(dst[n:], src[si:]); ok {
+ n += 6
+ si += 8
+ } else {
+ var ninc int
+ si, ninc, err = enc.decodeQuantum(dst[n:], src, si)
+ n += ninc
+ if err != nil {
+ return n, err
+ }
+ }
+ }
+
+ for ilen-si >= 4 && olen-n >= 4 {
+ if ok := enc.decode32(dst[n:], src[si:]); ok {
+ n += 3
+ si += 4
+ } else {
+ var ninc int
+ si, ninc, err = enc.decodeQuantum(dst[n:], src, si)
+ n += ninc
+ if err != nil {
+ return n, err
+ }
+ }
+ }
+
+ for si < len(src) {
+ var ninc int
+ si, ninc, err = enc.decodeQuantum(dst[n:], src, si)
+ n += ninc
+ if err != nil {
+ return n, err
+ }
+ }
+ return n, err
+}
+
+// decode32 tries to decode 4 base64 char into 3 bytes.
+// len(dst) and len(src) must both be >= 4.
+// Returns true if decode succeeded.
+func (enc *Encoding) decode32(dst, src []byte) bool {
+ var dn, n uint32
+ if n = uint32(enc.decodeMap[src[0]]); n == 0xff {
+ return false
+ }
+ dn |= n << 26
+ if n = uint32(enc.decodeMap[src[1]]); n == 0xff {
+ return false
+ }
+ dn |= n << 20
+ if n = uint32(enc.decodeMap[src[2]]); n == 0xff {
+ return false
+ }
+ dn |= n << 14
+ if n = uint32(enc.decodeMap[src[3]]); n == 0xff {
+ return false
+ }
+ dn |= n << 8
+
+ binary.BigEndian.PutUint32(dst, dn)
+ return true
+}
+
+// decode64 tries to decode 8 base64 char into 6 bytes.
+// len(dst) and len(src) must both be >= 8.
+// Returns true if decode succeeded.
+func (enc *Encoding) decode64(dst, src []byte) bool {
+ var dn, n uint64
+ if n = uint64(enc.decodeMap[src[0]]); n == 0xff {
+ return false
+ }
+ dn |= n << 58
+ if n = uint64(enc.decodeMap[src[1]]); n == 0xff {
+ return false
+ }
+ dn |= n << 52
+ if n = uint64(enc.decodeMap[src[2]]); n == 0xff {
+ return false
+ }
+ dn |= n << 46
+ if n = uint64(enc.decodeMap[src[3]]); n == 0xff {
+ return false
+ }
+ dn |= n << 40
+ if n = uint64(enc.decodeMap[src[4]]); n == 0xff {
+ return false
+ }
+ dn |= n << 34
+ if n = uint64(enc.decodeMap[src[5]]); n == 0xff {
+ return false
+ }
+ dn |= n << 28
+ if n = uint64(enc.decodeMap[src[6]]); n == 0xff {
+ return false
+ }
+ dn |= n << 22
+ if n = uint64(enc.decodeMap[src[7]]); n == 0xff {
+ return false
+ }
+ dn |= n << 16
+
+ binary.BigEndian.PutUint64(dst, dn)
+ return true
+}
+
type newlineFilteringReader struct {
wrapped io.Reader
}
diff --git a/src/encoding/base64/base64_test.go b/src/encoding/base64/base64_test.go
index 05011fb..9f5c493 100644
--- a/src/encoding/base64/base64_test.go
+++ b/src/encoding/base64/base64_test.go
@@ -152,12 +152,9 @@
for _, tt := range encodingTests {
encoded := tt.conv(p.encoded)
dbuf := make([]byte, tt.enc.DecodedLen(len(encoded)))
- count, end, err := tt.enc.decode(dbuf, []byte(encoded))
+ count, err := tt.enc.Decode(dbuf, []byte(encoded))
testEqual(t, "Decode(%q) = error %v, want %v", encoded, err, error(nil))
testEqual(t, "Decode(%q) = length %v, want %v", encoded, count, len(p.decoded))
- if len(encoded) > 0 {
- testEqual(t, "Decode(%q) = end %v, want %v", encoded, end, len(p.decoded)%3 != 0)
- }
testEqual(t, "Decode(%q) = %q, want %q", encoded, string(dbuf[0:count]), p.decoded)
dbuf, err = tt.enc.DecodeString(encoded)
diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go
index 8f485f1..0494a15 100644
--- a/src/go/build/deps_test.go
+++ b/src/go/build/deps_test.go
@@ -101,7 +101,7 @@
"crypto/cipher": {"L2", "crypto/subtle"},
"crypto/subtle": {},
"encoding/base32": {"L2"},
- "encoding/base64": {"L2"},
+ "encoding/base64": {"L2", "encoding/binary"},
"encoding/binary": {"L2", "reflect"},
"hash": {"L2"}, // interfaces
"hash/adler32": {"L2", "hash"},
To view, visit change 38632. To unsubscribe, or for help writing mail filters, visit settings.