From 38ed1bc0ecdc7fbeadbc7337dc44667688eac800 Mon Sep 17 00:00:00 2001
From: Garrett Bodley <garrett.bodley@gmail.com>
Date: Thu, 25 Jul 2024 22:25:00 -0400
Subject: [PATCH] blake2s: port blake2s_amd64.s to Avo

This implementation utilizes the same registers found in the reference
implementation, aiming to produce a minimal semantic diff between the
Avo-generated output and the original hand-written assembly.

To verify the Avo implementation, the reference and Avo-generated
assembly files are fed to `go tool asm`, capturing the debug output into
corresponding temp files. The debug output contains supplementary
metadata (line numbers, instruction offsets, and source file references)
that must be removed in order to obtain a semantic diff of the two
files. This is accomplished via a small utility script written in awk.

Commands used to verify Avo output:

GOROOT=$(go env GOROOT)
ASM_PATH="blake2s/blake2s_amd64.s"
REFERENCE="b2d3a6a4b4d36521cd7f653879cf6981e7c5c340"

go tool asm -o /dev/null -I "$GOROOT"/src/runtime -debug \
  <(git cat-file -p "$REFERENCE:$ASM_PATH") \
  > /tmp/reference.s

go tool asm -o /dev/null -I "$GOROOT"/src/runtime -debug \
  "$ASM_PATH" \
  > /tmp/avo.s

normalize(){
  awk '{
    $1=$2=$3="";
    print substr($0,4)
  }'
}

diff <(normalize < /tmp/reference.s) <(normalize < /tmp/avo.s)

Change-Id: Ica8bf9f0b42dc93714aa54e783fa74ed19e6b9f4
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/601216
Reviewed-by: Roland Shoemaker <roland@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Filippo Valsorda <filippo@golang.org>
---
 blake2s/_asm/blake2s_amd64_asm.go |  525 ++++++
 blake2s/_asm/go.mod               |   15 +
 blake2s/_asm/go.sum               |   12 +
 blake2s/blake2s_amd64.s           | 2571 ++++++++++++++++++++++++-----
 4 files changed, 2708 insertions(+), 415 deletions(-)
 create mode 100644 blake2s/_asm/blake2s_amd64_asm.go
 create mode 100644 blake2s/_asm/go.mod
 create mode 100644 blake2s/_asm/go.sum

diff --git a/blake2s/_asm/blake2s_amd64_asm.go b/blake2s/_asm/blake2s_amd64_asm.go
new file mode 100644
index 0000000000..48ddd6118a
--- /dev/null
+++ b/blake2s/_asm/blake2s_amd64_asm.go
@@ -0,0 +1,525 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	"github.com/mmcloughlin/avo/ir"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/blake2s"
+)
+
+//go:generate go run . -out ../blake2s_amd64.s -pkg blake2s
+
+func main() {
+	Package("golang.org/x/crypto/blake2s")
+	ConstraintExpr("amd64,gc,!purego")
+	hashBlocksSSE2()
+	hashBlocksSSSE3()
+	hashBlocksSSE4()
+	Generate()
+}
+
+func ROTL_SSE2(n uint64, t, v VecPhysical) {
+	MOVO(v, t)
+	PSLLL(Imm(n), t)
+	PSRLL(Imm(32-n), v)
+	PXOR(t, v)
+}
+
+func ROTL_SSSE3(c, v VecPhysical) {
+	PSHUFB(c, v)
+}
+
+func ROUND_SSE2(v0, v1, v2, v3 VecPhysical, m0, m1, m2, m3 Mem, t VecPhysical) {
+	PADDL(m0, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(16, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m1, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(24, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v1, v1)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v3, v3)
+	PADDL(m2, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(16, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m3, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSE2(24, t, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v3, v3)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v1, v1)
+}
+
+func ROUND_SSSE3(v0, v1, v2, v3 VecPhysical, m0, m1, m2, m3 Op, t, c16, c8 VecPhysical) {
+	PADDL(m0, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c16, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m1, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c8, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v1, v1)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v3, v3)
+	PADDL(m2, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c16, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(20, t, v1)
+	PADDL(m3, v0)
+	PADDL(v1, v0)
+	PXOR(v0, v3)
+	ROTL_SSSE3(c8, v3)
+	PADDL(v3, v2)
+	PXOR(v2, v1)
+	ROTL_SSE2(25, t, v1)
+	PSHUFL(Imm(0x39), v3, v3)
+	PSHUFL(Imm(0x4E), v2, v2)
+	PSHUFL(Imm(0x93), v1, v1)
+}
+
+func LOAD_MSG_SSE4(m0, m1, m2, m3 VecPhysical, src GPPhysical, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15 int) {
+	// Hack to get Avo to emit a MOVL instruction with a VecPhysical as the destination
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i0 * 4), m0}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i1*4), m0)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i2*4), m0)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i3*4), m0)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i4 * 4), m1}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i5*4), m1)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i6*4), m1)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i7*4), m1)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i8 * 4), m2}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i9*4), m2)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i10*4), m2)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i11*4), m2)
+	Instruction(&ir.Instruction{Opcode: "MOVL", Operands: []Op{Mem{Base: src}.Offset(i12 * 4), m3}})
+	PINSRD(Imm(1), Mem{Base: src}.Offset(i13*4), m3)
+	PINSRD(Imm(2), Mem{Base: src}.Offset(i14*4), m3)
+	PINSRD(Imm(3), Mem{Base: src}.Offset(i15*4), m3)
+}
+
+func PRECOMPUTE_MSG(dst GPPhysical, off int, src, R8, R9, R10, R11, R12, R13, R14, R15 GPPhysical) {
+	MOVQ(Mem{Base: src}.Offset(0*4), R8)
+	MOVQ(Mem{Base: src}.Offset(2*4), R9)
+	MOVQ(Mem{Base: src}.Offset(4*4), R10)
+	MOVQ(Mem{Base: src}.Offset(6*4), R11)
+	MOVQ(Mem{Base: src}.Offset(8*4), R12)
+	MOVQ(Mem{Base: src}.Offset(10*4), R13)
+	MOVQ(Mem{Base: src}.Offset(12*4), R14)
+	MOVQ(Mem{Base: src}.Offset(14*4), R15)
+
+	MOVL(R8L, Mem{Base: dst}.Offset(0*4+off+0))
+	MOVL(R8L, Mem{Base: dst}.Offset(9*4+off+64))
+	MOVL(R8L, Mem{Base: dst}.Offset(5*4+off+128))
+	MOVL(R8L, Mem{Base: dst}.Offset(14*4+off+192))
+	MOVL(R8L, Mem{Base: dst}.Offset(4*4+off+256))
+	MOVL(R8L, Mem{Base: dst}.Offset(2*4+off+320))
+	MOVL(R8L, Mem{Base: dst}.Offset(8*4+off+384))
+	MOVL(R8L, Mem{Base: dst}.Offset(12*4+off+448))
+	MOVL(R8L, Mem{Base: dst}.Offset(3*4+off+512))
+	MOVL(R8L, Mem{Base: dst}.Offset(15*4+off+576))
+	SHRQ(Imm(32), R8)
+	MOVL(R8L, Mem{Base: dst}.Offset(4*4+off+0))
+	MOVL(R8L, Mem{Base: dst}.Offset(8*4+off+64))
+	MOVL(R8L, Mem{Base: dst}.Offset(14*4+off+128))
+	MOVL(R8L, Mem{Base: dst}.Offset(5*4+off+192))
+	MOVL(R8L, Mem{Base: dst}.Offset(12*4+off+256))
+	MOVL(R8L, Mem{Base: dst}.Offset(11*4+off+320))
+	MOVL(R8L, Mem{Base: dst}.Offset(1*4+off+384))
+	MOVL(R8L, Mem{Base: dst}.Offset(6*4+off+448))
+	MOVL(R8L, Mem{Base: dst}.Offset(10*4+off+512))
+	MOVL(R8L, Mem{Base: dst}.Offset(3*4+off+576))
+
+	MOVL(R9L, Mem{Base: dst}.Offset(1*4+off+0))
+	MOVL(R9L, Mem{Base: dst}.Offset(13*4+off+64))
+	MOVL(R9L, Mem{Base: dst}.Offset(6*4+off+128))
+	MOVL(R9L, Mem{Base: dst}.Offset(8*4+off+192))
+	MOVL(R9L, Mem{Base: dst}.Offset(2*4+off+256))
+	MOVL(R9L, Mem{Base: dst}.Offset(0*4+off+320))
+	MOVL(R9L, Mem{Base: dst}.Offset(14*4+off+384))
+	MOVL(R9L, Mem{Base: dst}.Offset(11*4+off+448))
+	MOVL(R9L, Mem{Base: dst}.Offset(12*4+off+512))
+	MOVL(R9L, Mem{Base: dst}.Offset(4*4+off+576))
+	SHRQ(Imm(32), R9)
+	MOVL(R9L, Mem{Base: dst}.Offset(5*4+off+0))
+	MOVL(R9L, Mem{Base: dst}.Offset(15*4+off+64))
+	MOVL(R9L, Mem{Base: dst}.Offset(9*4+off+128))
+	MOVL(R9L, Mem{Base: dst}.Offset(1*4+off+192))
+	MOVL(R9L, Mem{Base: dst}.Offset(11*4+off+256))
+	MOVL(R9L, Mem{Base: dst}.Offset(7*4+off+320))
+	MOVL(R9L, Mem{Base: dst}.Offset(13*4+off+384))
+	MOVL(R9L, Mem{Base: dst}.Offset(3*4+off+448))
+	MOVL(R9L, Mem{Base: dst}.Offset(6*4+off+512))
+	MOVL(R9L, Mem{Base: dst}.Offset(10*4+off+576))
+
+	MOVL(R10L, Mem{Base: dst}.Offset(2*4+off+0))
+	MOVL(R10L, Mem{Base: dst}.Offset(1*4+off+64))
+	MOVL(R10L, Mem{Base: dst}.Offset(15*4+off+128))
+	MOVL(R10L, Mem{Base: dst}.Offset(10*4+off+192))
+	MOVL(R10L, Mem{Base: dst}.Offset(6*4+off+256))
+	MOVL(R10L, Mem{Base: dst}.Offset(8*4+off+320))
+	MOVL(R10L, Mem{Base: dst}.Offset(3*4+off+384))
+	MOVL(R10L, Mem{Base: dst}.Offset(13*4+off+448))
+	MOVL(R10L, Mem{Base: dst}.Offset(14*4+off+512))
+	MOVL(R10L, Mem{Base: dst}.Offset(5*4+off+576))
+	SHRQ(Imm(32), R10)
+	MOVL(R10L, Mem{Base: dst}.Offset(6*4+off+0))
+	MOVL(R10L, Mem{Base: dst}.Offset(11*4+off+64))
+	MOVL(R10L, Mem{Base: dst}.Offset(2*4+off+128))
+	MOVL(R10L, Mem{Base: dst}.Offset(9*4+off+192))
+	MOVL(R10L, Mem{Base: dst}.Offset(1*4+off+256))
+	MOVL(R10L, Mem{Base: dst}.Offset(13*4+off+320))
+	MOVL(R10L, Mem{Base: dst}.Offset(4*4+off+384))
+	MOVL(R10L, Mem{Base: dst}.Offset(8*4+off+448))
+	MOVL(R10L, Mem{Base: dst}.Offset(15*4+off+512))
+	MOVL(R10L, Mem{Base: dst}.Offset(7*4+off+576))
+
+	MOVL(R11L, Mem{Base: dst}.Offset(3*4+off+0))
+	MOVL(R11L, Mem{Base: dst}.Offset(7*4+off+64))
+	MOVL(R11L, Mem{Base: dst}.Offset(13*4+off+128))
+	MOVL(R11L, Mem{Base: dst}.Offset(12*4+off+192))
+	MOVL(R11L, Mem{Base: dst}.Offset(10*4+off+256))
+	MOVL(R11L, Mem{Base: dst}.Offset(1*4+off+320))
+	MOVL(R11L, Mem{Base: dst}.Offset(9*4+off+384))
+	MOVL(R11L, Mem{Base: dst}.Offset(14*4+off+448))
+	MOVL(R11L, Mem{Base: dst}.Offset(0*4+off+512))
+	MOVL(R11L, Mem{Base: dst}.Offset(6*4+off+576))
+	SHRQ(Imm(32), R11)
+	MOVL(R11L, Mem{Base: dst}.Offset(7*4+off+0))
+	MOVL(R11L, Mem{Base: dst}.Offset(14*4+off+64))
+	MOVL(R11L, Mem{Base: dst}.Offset(10*4+off+128))
+	MOVL(R11L, Mem{Base: dst}.Offset(0*4+off+192))
+	MOVL(R11L, Mem{Base: dst}.Offset(5*4+off+256))
+	MOVL(R11L, Mem{Base: dst}.Offset(9*4+off+320))
+	MOVL(R11L, Mem{Base: dst}.Offset(12*4+off+384))
+	MOVL(R11L, Mem{Base: dst}.Offset(1*4+off+448))
+	MOVL(R11L, Mem{Base: dst}.Offset(13*4+off+512))
+	MOVL(R11L, Mem{Base: dst}.Offset(2*4+off+576))
+
+	MOVL(R12L, Mem{Base: dst}.Offset(8*4+off+0))
+	MOVL(R12L, Mem{Base: dst}.Offset(5*4+off+64))
+	MOVL(R12L, Mem{Base: dst}.Offset(4*4+off+128))
+	MOVL(R12L, Mem{Base: dst}.Offset(15*4+off+192))
+	MOVL(R12L, Mem{Base: dst}.Offset(14*4+off+256))
+	MOVL(R12L, Mem{Base: dst}.Offset(3*4+off+320))
+	MOVL(R12L, Mem{Base: dst}.Offset(11*4+off+384))
+	MOVL(R12L, Mem{Base: dst}.Offset(10*4+off+448))
+	MOVL(R12L, Mem{Base: dst}.Offset(7*4+off+512))
+	MOVL(R12L, Mem{Base: dst}.Offset(1*4+off+576))
+	SHRQ(Imm(32), R12)
+	MOVL(R12L, Mem{Base: dst}.Offset(12*4+off+0))
+	MOVL(R12L, Mem{Base: dst}.Offset(2*4+off+64))
+	MOVL(R12L, Mem{Base: dst}.Offset(11*4+off+128))
+	MOVL(R12L, Mem{Base: dst}.Offset(4*4+off+192))
+	MOVL(R12L, Mem{Base: dst}.Offset(0*4+off+256))
+	MOVL(R12L, Mem{Base: dst}.Offset(15*4+off+320))
+	MOVL(R12L, Mem{Base: dst}.Offset(10*4+off+384))
+	MOVL(R12L, Mem{Base: dst}.Offset(7*4+off+448))
+	MOVL(R12L, Mem{Base: dst}.Offset(5*4+off+512))
+	MOVL(R12L, Mem{Base: dst}.Offset(9*4+off+576))
+
+	MOVL(R13L, Mem{Base: dst}.Offset(9*4+off+0))
+	MOVL(R13L, Mem{Base: dst}.Offset(4*4+off+64))
+	MOVL(R13L, Mem{Base: dst}.Offset(8*4+off+128))
+	MOVL(R13L, Mem{Base: dst}.Offset(13*4+off+192))
+	MOVL(R13L, Mem{Base: dst}.Offset(3*4+off+256))
+	MOVL(R13L, Mem{Base: dst}.Offset(5*4+off+320))
+	MOVL(R13L, Mem{Base: dst}.Offset(7*4+off+384))
+	MOVL(R13L, Mem{Base: dst}.Offset(15*4+off+448))
+	MOVL(R13L, Mem{Base: dst}.Offset(11*4+off+512))
+	MOVL(R13L, Mem{Base: dst}.Offset(0*4+off+576))
+	SHRQ(Imm(32), R13)
+	MOVL(R13L, Mem{Base: dst}.Offset(13*4+off+0))
+	MOVL(R13L, Mem{Base: dst}.Offset(10*4+off+64))
+	MOVL(R13L, Mem{Base: dst}.Offset(0*4+off+128))
+	MOVL(R13L, Mem{Base: dst}.Offset(3*4+off+192))
+	MOVL(R13L, Mem{Base: dst}.Offset(9*4+off+256))
+	MOVL(R13L, Mem{Base: dst}.Offset(6*4+off+320))
+	MOVL(R13L, Mem{Base: dst}.Offset(15*4+off+384))
+	MOVL(R13L, Mem{Base: dst}.Offset(4*4+off+448))
+	MOVL(R13L, Mem{Base: dst}.Offset(2*4+off+512))
+	MOVL(R13L, Mem{Base: dst}.Offset(12*4+off+576))
+
+	MOVL(R14L, Mem{Base: dst}.Offset(10*4+off+0))
+	MOVL(R14L, Mem{Base: dst}.Offset(12*4+off+64))
+	MOVL(R14L, Mem{Base: dst}.Offset(1*4+off+128))
+	MOVL(R14L, Mem{Base: dst}.Offset(6*4+off+192))
+	MOVL(R14L, Mem{Base: dst}.Offset(13*4+off+256))
+	MOVL(R14L, Mem{Base: dst}.Offset(4*4+off+320))
+	MOVL(R14L, Mem{Base: dst}.Offset(0*4+off+384))
+	MOVL(R14L, Mem{Base: dst}.Offset(2*4+off+448))
+	MOVL(R14L, Mem{Base: dst}.Offset(8*4+off+512))
+	MOVL(R14L, Mem{Base: dst}.Offset(14*4+off+576))
+	SHRQ(Imm(32), R14)
+	MOVL(R14L, Mem{Base: dst}.Offset(14*4+off+0))
+	MOVL(R14L, Mem{Base: dst}.Offset(3*4+off+64))
+	MOVL(R14L, Mem{Base: dst}.Offset(7*4+off+128))
+	MOVL(R14L, Mem{Base: dst}.Offset(2*4+off+192))
+	MOVL(R14L, Mem{Base: dst}.Offset(15*4+off+256))
+	MOVL(R14L, Mem{Base: dst}.Offset(12*4+off+320))
+	MOVL(R14L, Mem{Base: dst}.Offset(6*4+off+384))
+	MOVL(R14L, Mem{Base: dst}.Offset(0*4+off+448))
+	MOVL(R14L, Mem{Base: dst}.Offset(9*4+off+512))
+	MOVL(R14L, Mem{Base: dst}.Offset(11*4+off+576))
+
+	MOVL(R15L, Mem{Base: dst}.Offset(11*4+off+0))
+	MOVL(R15L, Mem{Base: dst}.Offset(0*4+off+64))
+	MOVL(R15L, Mem{Base: dst}.Offset(12*4+off+128))
+	MOVL(R15L, Mem{Base: dst}.Offset(7*4+off+192))
+	MOVL(R15L, Mem{Base: dst}.Offset(8*4+off+256))
+	MOVL(R15L, Mem{Base: dst}.Offset(14*4+off+320))
+	MOVL(R15L, Mem{Base: dst}.Offset(2*4+off+384))
+	MOVL(R15L, Mem{Base: dst}.Offset(5*4+off+448))
+	MOVL(R15L, Mem{Base: dst}.Offset(1*4+off+512))
+	MOVL(R15L, Mem{Base: dst}.Offset(13*4+off+576))
+	SHRQ(Imm(32), R15)
+	MOVL(R15L, Mem{Base: dst}.Offset(15*4+off+0))
+	MOVL(R15L, Mem{Base: dst}.Offset(6*4+off+64))
+	MOVL(R15L, Mem{Base: dst}.Offset(3*4+off+128))
+	MOVL(R15L, Mem{Base: dst}.Offset(11*4+off+192))
+	MOVL(R15L, Mem{Base: dst}.Offset(7*4+off+256))
+	MOVL(R15L, Mem{Base: dst}.Offset(10*4+off+320))
+	MOVL(R15L, Mem{Base: dst}.Offset(5*4+off+384))
+	MOVL(R15L, Mem{Base: dst}.Offset(9*4+off+448))
+	MOVL(R15L, Mem{Base: dst}.Offset(4*4+off+512))
+	MOVL(R15L, Mem{Base: dst}.Offset(8*4+off+576))
+}
+
+func BLAKE2s_SSE2() {
+	PRECOMPUTE_MSG(BP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15)
+	for i := 0; i < 10; i++ {
+		ROUND_SSE2(X4, X5, X6, X7, Mem{Base: BP}.Offset(16+64*i), Mem{Base: BP}.Offset(32+64*i), Mem{Base: BP}.Offset(48+64*i), Mem{Base: BP}.Offset(64+64*i), X8)
+	}
+}
+
+func BLAKE2s_SSSE3() {
+	PRECOMPUTE_MSG(BP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15)
+	for i := 0; i < 10; i++ {
+		ROUND_SSSE3(X4, X5, X6, X7, Mem{Base: BP}.Offset(16+64*i), Mem{Base: BP}.Offset(32+64*i), Mem{Base: BP}.Offset(48+64*i), Mem{Base: BP}.Offset(64+64*i), X8, X13, X14)
+	}
+}
+
+func BLAKE2s_SSE4() {
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0)
+	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+}
+
+func HASH_BLOCKS(h, c, flag, blocks_base, blocks_len Mem, BLAKE2s_FUNC func()) {
+	MOVQ(h, RAX)
+	MOVQ(c, RBX)
+	MOVL(flag, ECX)
+	MOVQ(blocks_base, RSI)
+	MOVQ(blocks_len, RDX)
+
+	MOVQ(RSP, RBP)
+	ADDQ(Imm(15), RBP)
+	ANDQ(I32(^15), RBP)
+
+	MOVQ(Mem{Base: BX}.Offset(0), R9)
+	MOVQ(R9, Mem{Base: BP}.Offset(0))
+	MOVQ(RCX, Mem{Base: BP}.Offset(8))
+
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: AX}.Offset(16), X1)
+
+	iv0 := iv0_DATA()
+	iv1 := iv1_DATA()
+	MOVOU(iv0, X2)
+	MOVOU(iv1, X3)
+
+	counter := counter_DATA()
+	rol16 := rol16_DATA()
+	rol8 := rol8_DATA()
+	MOVOU(counter, X12)
+	MOVOU(rol16, X13)
+	MOVOU(rol8, X14)
+	MOVO(Mem{Base: BP}.Offset(0), X15)
+
+	Label("loop")
+	MOVO(X0, X4)
+	MOVO(X1, X5)
+	MOVO(X2, X6)
+	MOVO(X3, X7)
+
+	PADDQ(X12, X15)
+	PXOR(X15, X7)
+
+	BLAKE2s_FUNC()
+
+	PXOR(X4, X0)
+	PXOR(X5, X1)
+	PXOR(X6, X0)
+	PXOR(X7, X1)
+
+	LEAQ(Mem{Base: SI}.Offset(64), RSI)
+	SUBQ(Imm(64), RDX)
+	JNE(LabelRef("loop"))
+
+	MOVO(X15, Mem{Base: BP}.Offset(0))
+	MOVQ(Mem{Base: BP}.Offset(0), R9)
+	MOVQ(R9, Mem{Base: BX}.Offset(0))
+
+	MOVOU(X0, Mem{Base: AX}.Offset(0))
+	MOVOU(X1, Mem{Base: AX}.Offset(16))
+}
+
+func hashBlocksSSE2() {
+	Implement("hashBlocksSSE2")
+	Attributes(0)
+	AllocLocal(672) // frame = 656 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSE2)
+	RET()
+}
+
+func hashBlocksSSSE3() {
+	Implement("hashBlocksSSSE3")
+	Attributes(0)
+	AllocLocal(672) // frame = 656 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSSE3)
+	RET()
+}
+
+func hashBlocksSSE4() {
+	Implement("hashBlocksSSE4")
+	Attributes(0)
+	AllocLocal(32) // frame = 16 + 16 byte alignment
+
+	h := NewParamAddr("h", 0)
+	c := NewParamAddr("c", 8)
+	flag := NewParamAddr("flag", 16)
+	blocks_base := NewParamAddr("blocks_base", 24)
+	blocks_len := NewParamAddr("blocks_len", 32)
+
+	HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_SSE4)
+	RET()
+}
+
+// ##~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~DATA SECTION~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~##
+
+var iv0_DATA_ptr, iv1_DATA_ptr, rol16_DATA_ptr, rol8_DATA_ptr, counter_DATA_ptr *Mem
+
+func iv0_DATA() Mem {
+	if iv0_DATA_ptr != nil {
+		return *iv0_DATA_ptr
+	}
+
+	iv0_DATA := GLOBL("iv0", NOPTR|RODATA)
+	iv0_DATA_ptr = &iv0_DATA
+	DATA(0x00, U32(0x6a09e667))
+	DATA(0x04, U32(0xbb67ae85))
+	DATA(0x08, U32(0x3c6ef372))
+	DATA(0x0c, U32(0xa54ff53a))
+	return iv0_DATA
+}
+
+func iv1_DATA() Mem {
+	if iv1_DATA_ptr != nil {
+		return *iv1_DATA_ptr
+	}
+
+	iv1_DATA := GLOBL("iv1", NOPTR|RODATA)
+	iv1_DATA_ptr = &iv1_DATA
+	DATA(0x00, U32(0x510e527f))
+	DATA(0x04, U32(0x9b05688c))
+	DATA(0x08, U32(0x1f83d9ab))
+	DATA(0x0c, U32(0x5be0cd19))
+	return iv1_DATA
+}
+
+func rol16_DATA() Mem {
+	if rol16_DATA_ptr != nil {
+		return *rol16_DATA_ptr
+	}
+
+	rol16_DATA := GLOBL("rol16", NOPTR|RODATA)
+	rol16_DATA_ptr = &rol16_DATA
+	DATA(0x00, U64(0x0504070601000302))
+	DATA(0x08, U64(0x0D0C0F0E09080B0A))
+	return rol16_DATA
+}
+
+func rol8_DATA() Mem {
+	if rol8_DATA_ptr != nil {
+		return *rol8_DATA_ptr
+	}
+
+	rol8_DATA := GLOBL("rol8", NOPTR|RODATA)
+	rol8_DATA_ptr = &rol8_DATA
+	DATA(0x00, U64(0x0407060500030201))
+	DATA(0x08, U64(0x0C0F0E0D080B0A09))
+	return rol8_DATA
+}
+
+func counter_DATA() Mem {
+	if counter_DATA_ptr != nil {
+		return *counter_DATA_ptr
+	}
+
+	counter_DATA := GLOBL("counter", NOPTR|RODATA)
+	counter_DATA_ptr = &counter_DATA
+	DATA(0x00, U64(0x0000000000000040))
+	DATA(0x08, U64(0x0000000000000000))
+	return counter_DATA
+}
diff --git a/blake2s/_asm/go.mod b/blake2s/_asm/go.mod
new file mode 100644
index 0000000000..9bb23e0eb1
--- /dev/null
+++ b/blake2s/_asm/go.mod
@@ -0,0 +1,15 @@
+module blake2s/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)
diff --git a/blake2s/_asm/go.sum b/blake2s/_asm/go.sum
new file mode 100644
index 0000000000..62ea9dfb70
--- /dev/null
+++ b/blake2s/_asm/go.sum
@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
diff --git a/blake2s/blake2s_amd64.s b/blake2s/blake2s_amd64.s
index fe4b818a33..57d510fc08 100644
--- a/blake2s/blake2s_amd64.s
+++ b/blake2s/blake2s_amd64.s
@@ -1,432 +1,2173 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run blake2s_amd64_asm.go -out ../blake2s_amd64.s -pkg blake2s. DO NOT EDIT.
 
 //go:build amd64 && gc && !purego
 
 #include "textflag.h"
 
-DATA iv0<>+0x00(SB)/4, $0x6a09e667
-DATA iv0<>+0x04(SB)/4, $0xbb67ae85
-DATA iv0<>+0x08(SB)/4, $0x3c6ef372
-DATA iv0<>+0x0c(SB)/4, $0xa54ff53a
-GLOBL iv0<>(SB), (NOPTR+RODATA), $16
-
-DATA iv1<>+0x00(SB)/4, $0x510e527f
-DATA iv1<>+0x04(SB)/4, $0x9b05688c
-DATA iv1<>+0x08(SB)/4, $0x1f83d9ab
-DATA iv1<>+0x0c(SB)/4, $0x5be0cd19
-GLOBL iv1<>(SB), (NOPTR+RODATA), $16
-
-DATA rol16<>+0x00(SB)/8, $0x0504070601000302
-DATA rol16<>+0x08(SB)/8, $0x0D0C0F0E09080B0A
-GLOBL rol16<>(SB), (NOPTR+RODATA), $16
-
-DATA rol8<>+0x00(SB)/8, $0x0407060500030201
-DATA rol8<>+0x08(SB)/8, $0x0C0F0E0D080B0A09
-GLOBL rol8<>(SB), (NOPTR+RODATA), $16
-
-DATA counter<>+0x00(SB)/8, $0x40
-DATA counter<>+0x08(SB)/8, $0x0
-GLOBL counter<>(SB), (NOPTR+RODATA), $16
-
-#define ROTL_SSE2(n, t, v) \
-	MOVO  v, t;       \
-	PSLLL $n, t;      \
-	PSRLL $(32-n), v; \
-	PXOR  t, v
-
-#define ROTL_SSSE3(c, v) \
-	PSHUFB c, v
-
-#define ROUND_SSE2(v0, v1, v2, v3, m0, m1, m2, m3, t) \
-	PADDL  m0, v0;        \
-	PADDL  v1, v0;        \
-	PXOR   v0, v3;        \
-	ROTL_SSE2(16, t, v3); \
-	PADDL  v3, v2;        \
-	PXOR   v2, v1;        \
-	ROTL_SSE2(20, t, v1); \
-	PADDL  m1, v0;        \
-	PADDL  v1, v0;        \
-	PXOR   v0, v3;        \
-	ROTL_SSE2(24, t, v3); \
-	PADDL  v3, v2;        \
-	PXOR   v2, v1;        \
-	ROTL_SSE2(25, t, v1); \
-	PSHUFL $0x39, v1, v1; \
-	PSHUFL $0x4E, v2, v2; \
-	PSHUFL $0x93, v3, v3; \
-	PADDL  m2, v0;        \
-	PADDL  v1, v0;        \
-	PXOR   v0, v3;        \
-	ROTL_SSE2(16, t, v3); \
-	PADDL  v3, v2;        \
-	PXOR   v2, v1;        \
-	ROTL_SSE2(20, t, v1); \
-	PADDL  m3, v0;        \
-	PADDL  v1, v0;        \
-	PXOR   v0, v3;        \
-	ROTL_SSE2(24, t, v3); \
-	PADDL  v3, v2;        \
-	PXOR   v2, v1;        \
-	ROTL_SSE2(25, t, v1); \
-	PSHUFL $0x39, v3, v3; \
-	PSHUFL $0x4E, v2, v2; \
-	PSHUFL $0x93, v1, v1
-
-#define ROUND_SSSE3(v0, v1, v2, v3, m0, m1, m2, m3, t, c16, c8) \
-	PADDL  m0, v0;        \
-	PADDL  v1, v0;        \
-	PXOR   v0, v3;        \
-	ROTL_SSSE3(c16, v3);  \
-	PADDL  v3, v2;        \
-	PXOR   v2, v1;        \
-	ROTL_SSE2(20, t, v1); \
-	PADDL  m1, v0;        \
-	PADDL  v1, v0;        \
-	PXOR   v0, v3;        \
-	ROTL_SSSE3(c8, v3);   \
-	PADDL  v3, v2;        \
-	PXOR   v2, v1;        \
-	ROTL_SSE2(25, t, v1); \
-	PSHUFL $0x39, v1, v1; \
-	PSHUFL $0x4E, v2, v2; \
-	PSHUFL $0x93, v3, v3; \
-	PADDL  m2, v0;        \
-	PADDL  v1, v0;        \
-	PXOR   v0, v3;        \
-	ROTL_SSSE3(c16, v3);  \
-	PADDL  v3, v2;        \
-	PXOR   v2, v1;        \
-	ROTL_SSE2(20, t, v1); \
-	PADDL  m3, v0;        \
-	PADDL  v1, v0;        \
-	PXOR   v0, v3;        \
-	ROTL_SSSE3(c8, v3);   \
-	PADDL  v3, v2;        \
-	PXOR   v2, v1;        \
-	ROTL_SSE2(25, t, v1); \
-	PSHUFL $0x39, v3, v3; \
-	PSHUFL $0x4E, v2, v2; \
-	PSHUFL $0x93, v1, v1
-
-
-#define LOAD_MSG_SSE4(m0, m1, m2, m3, src, i0, i1, i2, i3, i4, i5, i6, i7, i8, i9, i10, i11, i12, i13, i14, i15) \
-	MOVL   i0*4(src), m0;      \
-	PINSRD $1, i1*4(src), m0;  \
-	PINSRD $2, i2*4(src), m0;  \
-	PINSRD $3, i3*4(src), m0;  \
-	MOVL   i4*4(src), m1;      \
-	PINSRD $1, i5*4(src), m1;  \
-	PINSRD $2, i6*4(src), m1;  \
-	PINSRD $3, i7*4(src), m1;  \
-	MOVL   i8*4(src), m2;      \
-	PINSRD $1, i9*4(src), m2;  \
-	PINSRD $2, i10*4(src), m2; \
-	PINSRD $3, i11*4(src), m2; \
-	MOVL   i12*4(src), m3;     \
-	PINSRD $1, i13*4(src), m3; \
-	PINSRD $2, i14*4(src), m3; \
-	PINSRD $3, i15*4(src), m3
+// func hashBlocksSSE2(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
+// Requires: SSE2
+TEXT ·hashBlocksSSE2(SB), $672-48
+	MOVQ  h+0(FP), AX
+	MOVQ  c+8(FP), BX
+	MOVL  flag+16(FP), CX
+	MOVQ  blocks_base+24(FP), SI
+	MOVQ  blocks_len+32(FP), DX
+	MOVQ  SP, BP
+	ADDQ  $0x0f, BP
+	ANDQ  $-16, BP
+	MOVQ  (BX), R9
+	MOVQ  R9, (BP)
+	MOVQ  CX, 8(BP)
+	MOVOU (AX), X0
+	MOVOU 16(AX), X1
+	MOVOU iv0<>+0(SB), X2
+	MOVOU iv1<>+0(SB), X3
+	MOVOU counter<>+0(SB), X12
+	MOVOU rol16<>+0(SB), X13
+	MOVOU rol8<>+0(SB), X14
+	MOVO  (BP), X15
 
-#define PRECOMPUTE_MSG(dst, off, src, R8, R9, R10, R11, R12, R13, R14, R15) \
-	MOVQ 0*4(src), R8;           \
-	MOVQ 2*4(src), R9;           \
-	MOVQ 4*4(src), R10;          \
-	MOVQ 6*4(src), R11;          \
-	MOVQ 8*4(src), R12;          \
-	MOVQ 10*4(src), R13;         \
-	MOVQ 12*4(src), R14;         \
-	MOVQ 14*4(src), R15;         \
-	                             \
-	MOVL R8, 0*4+off+0(dst);     \
-	MOVL R8, 9*4+off+64(dst);    \
-	MOVL R8, 5*4+off+128(dst);   \
-	MOVL R8, 14*4+off+192(dst);  \
-	MOVL R8, 4*4+off+256(dst);   \
-	MOVL R8, 2*4+off+320(dst);   \
-	MOVL R8, 8*4+off+384(dst);   \
-	MOVL R8, 12*4+off+448(dst);  \
-	MOVL R8, 3*4+off+512(dst);   \
-	MOVL R8, 15*4+off+576(dst);  \
-	SHRQ $32, R8;                \
-	MOVL R8, 4*4+off+0(dst);     \
-	MOVL R8, 8*4+off+64(dst);    \
-	MOVL R8, 14*4+off+128(dst);  \
-	MOVL R8, 5*4+off+192(dst);   \
-	MOVL R8, 12*4+off+256(dst);  \
-	MOVL R8, 11*4+off+320(dst);  \
-	MOVL R8, 1*4+off+384(dst);   \
-	MOVL R8, 6*4+off+448(dst);   \
-	MOVL R8, 10*4+off+512(dst);  \
-	MOVL R8, 3*4+off+576(dst);   \
-	                             \
-	MOVL R9, 1*4+off+0(dst);     \
-	MOVL R9, 13*4+off+64(dst);   \
-	MOVL R9, 6*4+off+128(dst);   \
-	MOVL R9, 8*4+off+192(dst);   \
-	MOVL R9, 2*4+off+256(dst);   \
-	MOVL R9, 0*4+off+320(dst);   \
-	MOVL R9, 14*4+off+384(dst);  \
-	MOVL R9, 11*4+off+448(dst);  \
-	MOVL R9, 12*4+off+512(dst);  \
-	MOVL R9, 4*4+off+576(dst);   \
-	SHRQ $32, R9;                \
-	MOVL R9, 5*4+off+0(dst);     \
-	MOVL R9, 15*4+off+64(dst);   \
-	MOVL R9, 9*4+off+128(dst);   \
-	MOVL R9, 1*4+off+192(dst);   \
-	MOVL R9, 11*4+off+256(dst);  \
-	MOVL R9, 7*4+off+320(dst);   \
-	MOVL R9, 13*4+off+384(dst);  \
-	MOVL R9, 3*4+off+448(dst);   \
-	MOVL R9, 6*4+off+512(dst);   \
-	MOVL R9, 10*4+off+576(dst);  \
-	                             \
-	MOVL R10, 2*4+off+0(dst);    \
-	MOVL R10, 1*4+off+64(dst);   \
-	MOVL R10, 15*4+off+128(dst); \
-	MOVL R10, 10*4+off+192(dst); \
-	MOVL R10, 6*4+off+256(dst);  \
-	MOVL R10, 8*4+off+320(dst);  \
-	MOVL R10, 3*4+off+384(dst);  \
-	MOVL R10, 13*4+off+448(dst); \
-	MOVL R10, 14*4+off+512(dst); \
-	MOVL R10, 5*4+off+576(dst);  \
-	SHRQ $32, R10;               \
-	MOVL R10, 6*4+off+0(dst);    \
-	MOVL R10, 11*4+off+64(dst);  \
-	MOVL R10, 2*4+off+128(dst);  \
-	MOVL R10, 9*4+off+192(dst);  \
-	MOVL R10, 1*4+off+256(dst);  \
-	MOVL R10, 13*4+off+320(dst); \
-	MOVL R10, 4*4+off+384(dst);  \
-	MOVL R10, 8*4+off+448(dst);  \
-	MOVL R10, 15*4+off+512(dst); \
-	MOVL R10, 7*4+off+576(dst);  \
-	                             \
-	MOVL R11, 3*4+off+0(dst);    \
-	MOVL R11, 7*4+off+64(dst);   \
-	MOVL R11, 13*4+off+128(dst); \
-	MOVL R11, 12*4+off+192(dst); \
-	MOVL R11, 10*4+off+256(dst); \
-	MOVL R11, 1*4+off+320(dst);  \
-	MOVL R11, 9*4+off+384(dst);  \
-	MOVL R11, 14*4+off+448(dst); \
-	MOVL R11, 0*4+off+512(dst);  \
-	MOVL R11, 6*4+off+576(dst);  \
-	SHRQ $32, R11;               \
-	MOVL R11, 7*4+off+0(dst);    \
-	MOVL R11, 14*4+off+64(dst);  \
-	MOVL R11, 10*4+off+128(dst); \
-	MOVL R11, 0*4+off+192(dst);  \
-	MOVL R11, 5*4+off+256(dst);  \
-	MOVL R11, 9*4+off+320(dst);  \
-	MOVL R11, 12*4+off+384(dst); \
-	MOVL R11, 1*4+off+448(dst);  \
-	MOVL R11, 13*4+off+512(dst); \
-	MOVL R11, 2*4+off+576(dst);  \
-	                             \
-	MOVL R12, 8*4+off+0(dst);    \
-	MOVL R12, 5*4+off+64(dst);   \
-	MOVL R12, 4*4+off+128(dst);  \
-	MOVL R12, 15*4+off+192(dst); \
-	MOVL R12, 14*4+off+256(dst); \
-	MOVL R12, 3*4+off+320(dst);  \
-	MOVL R12, 11*4+off+384(dst); \
-	MOVL R12, 10*4+off+448(dst); \
-	MOVL R12, 7*4+off+512(dst);  \
-	MOVL R12, 1*4+off+576(dst);  \
-	SHRQ $32, R12;               \
-	MOVL R12, 12*4+off+0(dst);   \
-	MOVL R12, 2*4+off+64(dst);   \
-	MOVL R12, 11*4+off+128(dst); \
-	MOVL R12, 4*4+off+192(dst);  \
-	MOVL R12, 0*4+off+256(dst);  \
-	MOVL R12, 15*4+off+320(dst); \
-	MOVL R12, 10*4+off+384(dst); \
-	MOVL R12, 7*4+off+448(dst);  \
-	MOVL R12, 5*4+off+512(dst);  \
-	MOVL R12, 9*4+off+576(dst);  \
-	                             \
-	MOVL R13, 9*4+off+0(dst);    \
-	MOVL R13, 4*4+off+64(dst);   \
-	MOVL R13, 8*4+off+128(dst);  \
-	MOVL R13, 13*4+off+192(dst); \
-	MOVL R13, 3*4+off+256(dst);  \
-	MOVL R13, 5*4+off+320(dst);  \
-	MOVL R13, 7*4+off+384(dst);  \
-	MOVL R13, 15*4+off+448(dst); \
-	MOVL R13, 11*4+off+512(dst); \
-	MOVL R13, 0*4+off+576(dst);  \
-	SHRQ $32, R13;               \
-	MOVL R13, 13*4+off+0(dst);   \
-	MOVL R13, 10*4+off+64(dst);  \
-	MOVL R13, 0*4+off+128(dst);  \
-	MOVL R13, 3*4+off+192(dst);  \
-	MOVL R13, 9*4+off+256(dst);  \
-	MOVL R13, 6*4+off+320(dst);  \
-	MOVL R13, 15*4+off+384(dst); \
-	MOVL R13, 4*4+off+448(dst);  \
-	MOVL R13, 2*4+off+512(dst);  \
-	MOVL R13, 12*4+off+576(dst); \
-	                             \
-	MOVL R14, 10*4+off+0(dst);   \
-	MOVL R14, 12*4+off+64(dst);  \
-	MOVL R14, 1*4+off+128(dst);  \
-	MOVL R14, 6*4+off+192(dst);  \
-	MOVL R14, 13*4+off+256(dst); \
-	MOVL R14, 4*4+off+320(dst);  \
-	MOVL R14, 0*4+off+384(dst);  \
-	MOVL R14, 2*4+off+448(dst);  \
-	MOVL R14, 8*4+off+512(dst);  \
-	MOVL R14, 14*4+off+576(dst); \
-	SHRQ $32, R14;               \
-	MOVL R14, 14*4+off+0(dst);   \
-	MOVL R14, 3*4+off+64(dst);   \
-	MOVL R14, 7*4+off+128(dst);  \
-	MOVL R14, 2*4+off+192(dst);  \
-	MOVL R14, 15*4+off+256(dst); \
-	MOVL R14, 12*4+off+320(dst); \
-	MOVL R14, 6*4+off+384(dst);  \
-	MOVL R14, 0*4+off+448(dst);  \
-	MOVL R14, 9*4+off+512(dst);  \
-	MOVL R14, 11*4+off+576(dst); \
-	                             \
-	MOVL R15, 11*4+off+0(dst);   \
-	MOVL R15, 0*4+off+64(dst);   \
-	MOVL R15, 12*4+off+128(dst); \
-	MOVL R15, 7*4+off+192(dst);  \
-	MOVL R15, 8*4+off+256(dst);  \
-	MOVL R15, 14*4+off+320(dst); \
-	MOVL R15, 2*4+off+384(dst);  \
-	MOVL R15, 5*4+off+448(dst);  \
-	MOVL R15, 1*4+off+512(dst);  \
-	MOVL R15, 13*4+off+576(dst); \
-	SHRQ $32, R15;               \
-	MOVL R15, 15*4+off+0(dst);   \
-	MOVL R15, 6*4+off+64(dst);   \
-	MOVL R15, 3*4+off+128(dst);  \
-	MOVL R15, 11*4+off+192(dst); \
-	MOVL R15, 7*4+off+256(dst);  \
-	MOVL R15, 10*4+off+320(dst); \
-	MOVL R15, 5*4+off+384(dst);  \
-	MOVL R15, 9*4+off+448(dst);  \
-	MOVL R15, 4*4+off+512(dst);  \
-	MOVL R15, 8*4+off+576(dst)
+loop:
+	MOVO   X0, X4
+	MOVO   X1, X5
+	MOVO   X2, X6
+	MOVO   X3, X7
+	PADDQ  X12, X15
+	PXOR   X15, X7
+	MOVQ   (SI), R8
+	MOVQ   8(SI), R9
+	MOVQ   16(SI), R10
+	MOVQ   24(SI), R11
+	MOVQ   32(SI), R12
+	MOVQ   40(SI), R13
+	MOVQ   48(SI), R14
+	MOVQ   56(SI), R15
+	MOVL   R8, 16(BP)
+	MOVL   R8, 116(BP)
+	MOVL   R8, 164(BP)
+	MOVL   R8, 264(BP)
+	MOVL   R8, 288(BP)
+	MOVL   R8, 344(BP)
+	MOVL   R8, 432(BP)
+	MOVL   R8, 512(BP)
+	MOVL   R8, 540(BP)
+	MOVL   R8, 652(BP)
+	SHRQ   $0x20, R8
+	MOVL   R8, 32(BP)
+	MOVL   R8, 112(BP)
+	MOVL   R8, 200(BP)
+	MOVL   R8, 228(BP)
+	MOVL   R8, 320(BP)
+	MOVL   R8, 380(BP)
+	MOVL   R8, 404(BP)
+	MOVL   R8, 488(BP)
+	MOVL   R8, 568(BP)
+	MOVL   R8, 604(BP)
+	MOVL   R9, 20(BP)
+	MOVL   R9, 132(BP)
+	MOVL   R9, 168(BP)
+	MOVL   R9, 240(BP)
+	MOVL   R9, 280(BP)
+	MOVL   R9, 336(BP)
+	MOVL   R9, 456(BP)
+	MOVL   R9, 508(BP)
+	MOVL   R9, 576(BP)
+	MOVL   R9, 608(BP)
+	SHRQ   $0x20, R9
+	MOVL   R9, 36(BP)
+	MOVL   R9, 140(BP)
+	MOVL   R9, 180(BP)
+	MOVL   R9, 212(BP)
+	MOVL   R9, 316(BP)
+	MOVL   R9, 364(BP)
+	MOVL   R9, 452(BP)
+	MOVL   R9, 476(BP)
+	MOVL   R9, 552(BP)
+	MOVL   R9, 632(BP)
+	MOVL   R10, 24(BP)
+	MOVL   R10, 84(BP)
+	MOVL   R10, 204(BP)
+	MOVL   R10, 248(BP)
+	MOVL   R10, 296(BP)
+	MOVL   R10, 368(BP)
+	MOVL   R10, 412(BP)
+	MOVL   R10, 516(BP)
+	MOVL   R10, 584(BP)
+	MOVL   R10, 612(BP)
+	SHRQ   $0x20, R10
+	MOVL   R10, 40(BP)
+	MOVL   R10, 124(BP)
+	MOVL   R10, 152(BP)
+	MOVL   R10, 244(BP)
+	MOVL   R10, 276(BP)
+	MOVL   R10, 388(BP)
+	MOVL   R10, 416(BP)
+	MOVL   R10, 496(BP)
+	MOVL   R10, 588(BP)
+	MOVL   R10, 620(BP)
+	MOVL   R11, 28(BP)
+	MOVL   R11, 108(BP)
+	MOVL   R11, 196(BP)
+	MOVL   R11, 256(BP)
+	MOVL   R11, 312(BP)
+	MOVL   R11, 340(BP)
+	MOVL   R11, 436(BP)
+	MOVL   R11, 520(BP)
+	MOVL   R11, 528(BP)
+	MOVL   R11, 616(BP)
+	SHRQ   $0x20, R11
+	MOVL   R11, 44(BP)
+	MOVL   R11, 136(BP)
+	MOVL   R11, 184(BP)
+	MOVL   R11, 208(BP)
+	MOVL   R11, 292(BP)
+	MOVL   R11, 372(BP)
+	MOVL   R11, 448(BP)
+	MOVL   R11, 468(BP)
+	MOVL   R11, 580(BP)
+	MOVL   R11, 600(BP)
+	MOVL   R12, 48(BP)
+	MOVL   R12, 100(BP)
+	MOVL   R12, 160(BP)
+	MOVL   R12, 268(BP)
+	MOVL   R12, 328(BP)
+	MOVL   R12, 348(BP)
+	MOVL   R12, 444(BP)
+	MOVL   R12, 504(BP)
+	MOVL   R12, 556(BP)
+	MOVL   R12, 596(BP)
+	SHRQ   $0x20, R12
+	MOVL   R12, 64(BP)
+	MOVL   R12, 88(BP)
+	MOVL   R12, 188(BP)
+	MOVL   R12, 224(BP)
+	MOVL   R12, 272(BP)
+	MOVL   R12, 396(BP)
+	MOVL   R12, 440(BP)
+	MOVL   R12, 492(BP)
+	MOVL   R12, 548(BP)
+	MOVL   R12, 628(BP)
+	MOVL   R13, 52(BP)
+	MOVL   R13, 96(BP)
+	MOVL   R13, 176(BP)
+	MOVL   R13, 260(BP)
+	MOVL   R13, 284(BP)
+	MOVL   R13, 356(BP)
+	MOVL   R13, 428(BP)
+	MOVL   R13, 524(BP)
+	MOVL   R13, 572(BP)
+	MOVL   R13, 592(BP)
+	SHRQ   $0x20, R13
+	MOVL   R13, 68(BP)
+	MOVL   R13, 120(BP)
+	MOVL   R13, 144(BP)
+	MOVL   R13, 220(BP)
+	MOVL   R13, 308(BP)
+	MOVL   R13, 360(BP)
+	MOVL   R13, 460(BP)
+	MOVL   R13, 480(BP)
+	MOVL   R13, 536(BP)
+	MOVL   R13, 640(BP)
+	MOVL   R14, 56(BP)
+	MOVL   R14, 128(BP)
+	MOVL   R14, 148(BP)
+	MOVL   R14, 232(BP)
+	MOVL   R14, 324(BP)
+	MOVL   R14, 352(BP)
+	MOVL   R14, 400(BP)
+	MOVL   R14, 472(BP)
+	MOVL   R14, 560(BP)
+	MOVL   R14, 648(BP)
+	SHRQ   $0x20, R14
+	MOVL   R14, 72(BP)
+	MOVL   R14, 92(BP)
+	MOVL   R14, 172(BP)
+	MOVL   R14, 216(BP)
+	MOVL   R14, 332(BP)
+	MOVL   R14, 384(BP)
+	MOVL   R14, 424(BP)
+	MOVL   R14, 464(BP)
+	MOVL   R14, 564(BP)
+	MOVL   R14, 636(BP)
+	MOVL   R15, 60(BP)
+	MOVL   R15, 80(BP)
+	MOVL   R15, 192(BP)
+	MOVL   R15, 236(BP)
+	MOVL   R15, 304(BP)
+	MOVL   R15, 392(BP)
+	MOVL   R15, 408(BP)
+	MOVL   R15, 484(BP)
+	MOVL   R15, 532(BP)
+	MOVL   R15, 644(BP)
+	SHRQ   $0x20, R15
+	MOVL   R15, 76(BP)
+	MOVL   R15, 104(BP)
+	MOVL   R15, 156(BP)
+	MOVL   R15, 252(BP)
+	MOVL   R15, 300(BP)
+	MOVL   R15, 376(BP)
+	MOVL   R15, 420(BP)
+	MOVL   R15, 500(BP)
+	MOVL   R15, 544(BP)
+	MOVL   R15, 624(BP)
+	PADDL  16(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  32(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  48(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  64(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  80(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  96(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  112(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  128(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  144(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  160(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  176(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  192(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  208(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  224(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  240(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  256(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  272(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  288(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  304(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  320(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  336(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  352(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  368(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  384(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  400(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  416(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  432(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  448(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  464(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  480(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  496(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  512(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  528(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  544(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  560(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  576(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  592(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  608(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  624(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x10, X8
+	PSRLL  $0x10, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  640(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	MOVO   X7, X8
+	PSLLL  $0x18, X8
+	PSRLL  $0x08, X7
+	PXOR   X8, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PXOR   X4, X0
+	PXOR   X5, X1
+	PXOR   X6, X0
+	PXOR   X7, X1
+	LEAQ   64(SI), SI
+	SUBQ   $0x40, DX
+	JNE    loop
+	MOVO   X15, (BP)
+	MOVQ   (BP), R9
+	MOVQ   R9, (BX)
+	MOVOU  X0, (AX)
+	MOVOU  X1, 16(AX)
+	RET
 
-#define BLAKE2s_SSE2() \
-	PRECOMPUTE_MSG(BP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15);               \
-	ROUND_SSE2(X4, X5, X6, X7, 16(BP), 32(BP), 48(BP), 64(BP), X8);                 \
-	ROUND_SSE2(X4, X5, X6, X7, 16+64(BP), 32+64(BP), 48+64(BP), 64+64(BP), X8);     \
-	ROUND_SSE2(X4, X5, X6, X7, 16+128(BP), 32+128(BP), 48+128(BP), 64+128(BP), X8); \
-	ROUND_SSE2(X4, X5, X6, X7, 16+192(BP), 32+192(BP), 48+192(BP), 64+192(BP), X8); \
-	ROUND_SSE2(X4, X5, X6, X7, 16+256(BP), 32+256(BP), 48+256(BP), 64+256(BP), X8); \
-	ROUND_SSE2(X4, X5, X6, X7, 16+320(BP), 32+320(BP), 48+320(BP), 64+320(BP), X8); \
-	ROUND_SSE2(X4, X5, X6, X7, 16+384(BP), 32+384(BP), 48+384(BP), 64+384(BP), X8); \
-	ROUND_SSE2(X4, X5, X6, X7, 16+448(BP), 32+448(BP), 48+448(BP), 64+448(BP), X8); \
-	ROUND_SSE2(X4, X5, X6, X7, 16+512(BP), 32+512(BP), 48+512(BP), 64+512(BP), X8); \
-	ROUND_SSE2(X4, X5, X6, X7, 16+576(BP), 32+576(BP), 48+576(BP), 64+576(BP), X8)
+DATA iv0<>+0(SB)/4, $0x6a09e667
+DATA iv0<>+4(SB)/4, $0xbb67ae85
+DATA iv0<>+8(SB)/4, $0x3c6ef372
+DATA iv0<>+12(SB)/4, $0xa54ff53a
+GLOBL iv0<>(SB), RODATA|NOPTR, $16
 
-#define BLAKE2s_SSSE3() \
-	PRECOMPUTE_MSG(BP, 16, SI, R8, R9, R10, R11, R12, R13, R14, R15);                          \
-	ROUND_SSSE3(X4, X5, X6, X7, 16(BP), 32(BP), 48(BP), 64(BP), X8, X13, X14);                 \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+64(BP), 32+64(BP), 48+64(BP), 64+64(BP), X8, X13, X14);     \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+128(BP), 32+128(BP), 48+128(BP), 64+128(BP), X8, X13, X14); \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+192(BP), 32+192(BP), 48+192(BP), 64+192(BP), X8, X13, X14); \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+256(BP), 32+256(BP), 48+256(BP), 64+256(BP), X8, X13, X14); \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+320(BP), 32+320(BP), 48+320(BP), 64+320(BP), X8, X13, X14); \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+384(BP), 32+384(BP), 48+384(BP), 64+384(BP), X8, X13, X14); \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+448(BP), 32+448(BP), 48+448(BP), 64+448(BP), X8, X13, X14); \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+512(BP), 32+512(BP), 48+512(BP), 64+512(BP), X8, X13, X14); \
-	ROUND_SSSE3(X4, X5, X6, X7, 16+576(BP), 32+576(BP), 48+576(BP), 64+576(BP), X8, X13, X14)
+DATA iv1<>+0(SB)/4, $0x510e527f
+DATA iv1<>+4(SB)/4, $0x9b05688c
+DATA iv1<>+8(SB)/4, $0x1f83d9ab
+DATA iv1<>+12(SB)/4, $0x5be0cd19
+GLOBL iv1<>(SB), RODATA|NOPTR, $16
 
-#define BLAKE2s_SSE4() \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6, 1, 0, 11, 5, 12, 2, 7, 3); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13, 10, 3, 7, 9, 14, 6, 1, 4); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14, 2, 5, 4, 15, 6, 10, 0, 8); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15, 14, 11, 6, 3, 1, 12, 8, 13); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3, 4, 7, 15, 1, 13, 5, 14, 9); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10, 0, 6, 9, 8, 7, 3, 2, 11); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9, 5, 15, 8, 2, 0, 4, 6, 10); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8, 12, 13, 1, 10, 2, 7, 4, 5); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14);                               \
-	LOAD_MSG_SSE4(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5, 15, 9, 3, 13, 11, 14, 12, 0); \
-	ROUND_SSSE3(X4, X5, X6, X7, X8, X9, X10, X11, X8, X13, X14)
+DATA counter<>+0(SB)/8, $0x0000000000000040
+DATA counter<>+8(SB)/8, $0x0000000000000000
+GLOBL counter<>(SB), RODATA|NOPTR, $16
 
-#define HASH_BLOCKS(h, c, flag, blocks_base, blocks_len, BLAKE2s_FUNC) \
-	MOVQ  h, AX;                   \
-	MOVQ  c, BX;                   \
-	MOVL  flag, CX;                \
-	MOVQ  blocks_base, SI;         \
-	MOVQ  blocks_len, DX;          \
-	                               \
-	MOVQ  SP, BP;                  \
-	ADDQ  $15, BP;                 \
-	ANDQ  $~15, BP;                \
-	                               \
-	MOVQ  0(BX), R9;               \
-	MOVQ  R9, 0(BP);               \
-	MOVQ  CX, 8(BP);               \
-	                               \
-	MOVOU 0(AX), X0;               \
-	MOVOU 16(AX), X1;              \
-	MOVOU iv0<>(SB), X2;           \
-	MOVOU iv1<>(SB), X3            \
-	                               \
-	MOVOU counter<>(SB), X12;      \
-	MOVOU rol16<>(SB), X13;        \
-	MOVOU rol8<>(SB), X14;         \
-	MOVO  0(BP), X15;              \
-	                               \
-	loop:                          \
-	MOVO  X0, X4;                  \
-	MOVO  X1, X5;                  \
-	MOVO  X2, X6;                  \
-	MOVO  X3, X7;                  \
-	                               \
-	PADDQ X12, X15;                \
-	PXOR  X15, X7;                 \
-	                               \
-	BLAKE2s_FUNC();                \
-	                               \
-	PXOR  X4, X0;                  \
-	PXOR  X5, X1;                  \
-	PXOR  X6, X0;                  \
-	PXOR  X7, X1;                  \
-	                               \
-	LEAQ  64(SI), SI;              \
-	SUBQ  $64, DX;                 \
-	JNE   loop;                    \
-	                               \
-	MOVO  X15, 0(BP);              \
-	MOVQ  0(BP), R9;               \
-	MOVQ  R9, 0(BX);               \
-	                               \
-	MOVOU X0, 0(AX);               \
-	MOVOU X1, 16(AX)
+DATA rol16<>+0(SB)/8, $0x0504070601000302
+DATA rol16<>+8(SB)/8, $0x0d0c0f0e09080b0a
+GLOBL rol16<>(SB), RODATA|NOPTR, $16
 
-// func hashBlocksSSE2(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
-TEXT ·hashBlocksSSE2(SB), 0, $672-48 // frame = 656 + 16 byte alignment
-	HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSE2)
-	RET
+DATA rol8<>+0(SB)/8, $0x0407060500030201
+DATA rol8<>+8(SB)/8, $0x0c0f0e0d080b0a09
+GLOBL rol8<>(SB), RODATA|NOPTR, $16
 
 // func hashBlocksSSSE3(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
-TEXT ·hashBlocksSSSE3(SB), 0, $672-48 // frame = 656 + 16 byte alignment
-	HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSSE3)
+// Requires: SSE2, SSSE3
+TEXT ·hashBlocksSSSE3(SB), $672-48
+	MOVQ  h+0(FP), AX
+	MOVQ  c+8(FP), BX
+	MOVL  flag+16(FP), CX
+	MOVQ  blocks_base+24(FP), SI
+	MOVQ  blocks_len+32(FP), DX
+	MOVQ  SP, BP
+	ADDQ  $0x0f, BP
+	ANDQ  $-16, BP
+	MOVQ  (BX), R9
+	MOVQ  R9, (BP)
+	MOVQ  CX, 8(BP)
+	MOVOU (AX), X0
+	MOVOU 16(AX), X1
+	MOVOU iv0<>+0(SB), X2
+	MOVOU iv1<>+0(SB), X3
+	MOVOU counter<>+0(SB), X12
+	MOVOU rol16<>+0(SB), X13
+	MOVOU rol8<>+0(SB), X14
+	MOVO  (BP), X15
+
+loop:
+	MOVO   X0, X4
+	MOVO   X1, X5
+	MOVO   X2, X6
+	MOVO   X3, X7
+	PADDQ  X12, X15
+	PXOR   X15, X7
+	MOVQ   (SI), R8
+	MOVQ   8(SI), R9
+	MOVQ   16(SI), R10
+	MOVQ   24(SI), R11
+	MOVQ   32(SI), R12
+	MOVQ   40(SI), R13
+	MOVQ   48(SI), R14
+	MOVQ   56(SI), R15
+	MOVL   R8, 16(BP)
+	MOVL   R8, 116(BP)
+	MOVL   R8, 164(BP)
+	MOVL   R8, 264(BP)
+	MOVL   R8, 288(BP)
+	MOVL   R8, 344(BP)
+	MOVL   R8, 432(BP)
+	MOVL   R8, 512(BP)
+	MOVL   R8, 540(BP)
+	MOVL   R8, 652(BP)
+	SHRQ   $0x20, R8
+	MOVL   R8, 32(BP)
+	MOVL   R8, 112(BP)
+	MOVL   R8, 200(BP)
+	MOVL   R8, 228(BP)
+	MOVL   R8, 320(BP)
+	MOVL   R8, 380(BP)
+	MOVL   R8, 404(BP)
+	MOVL   R8, 488(BP)
+	MOVL   R8, 568(BP)
+	MOVL   R8, 604(BP)
+	MOVL   R9, 20(BP)
+	MOVL   R9, 132(BP)
+	MOVL   R9, 168(BP)
+	MOVL   R9, 240(BP)
+	MOVL   R9, 280(BP)
+	MOVL   R9, 336(BP)
+	MOVL   R9, 456(BP)
+	MOVL   R9, 508(BP)
+	MOVL   R9, 576(BP)
+	MOVL   R9, 608(BP)
+	SHRQ   $0x20, R9
+	MOVL   R9, 36(BP)
+	MOVL   R9, 140(BP)
+	MOVL   R9, 180(BP)
+	MOVL   R9, 212(BP)
+	MOVL   R9, 316(BP)
+	MOVL   R9, 364(BP)
+	MOVL   R9, 452(BP)
+	MOVL   R9, 476(BP)
+	MOVL   R9, 552(BP)
+	MOVL   R9, 632(BP)
+	MOVL   R10, 24(BP)
+	MOVL   R10, 84(BP)
+	MOVL   R10, 204(BP)
+	MOVL   R10, 248(BP)
+	MOVL   R10, 296(BP)
+	MOVL   R10, 368(BP)
+	MOVL   R10, 412(BP)
+	MOVL   R10, 516(BP)
+	MOVL   R10, 584(BP)
+	MOVL   R10, 612(BP)
+	SHRQ   $0x20, R10
+	MOVL   R10, 40(BP)
+	MOVL   R10, 124(BP)
+	MOVL   R10, 152(BP)
+	MOVL   R10, 244(BP)
+	MOVL   R10, 276(BP)
+	MOVL   R10, 388(BP)
+	MOVL   R10, 416(BP)
+	MOVL   R10, 496(BP)
+	MOVL   R10, 588(BP)
+	MOVL   R10, 620(BP)
+	MOVL   R11, 28(BP)
+	MOVL   R11, 108(BP)
+	MOVL   R11, 196(BP)
+	MOVL   R11, 256(BP)
+	MOVL   R11, 312(BP)
+	MOVL   R11, 340(BP)
+	MOVL   R11, 436(BP)
+	MOVL   R11, 520(BP)
+	MOVL   R11, 528(BP)
+	MOVL   R11, 616(BP)
+	SHRQ   $0x20, R11
+	MOVL   R11, 44(BP)
+	MOVL   R11, 136(BP)
+	MOVL   R11, 184(BP)
+	MOVL   R11, 208(BP)
+	MOVL   R11, 292(BP)
+	MOVL   R11, 372(BP)
+	MOVL   R11, 448(BP)
+	MOVL   R11, 468(BP)
+	MOVL   R11, 580(BP)
+	MOVL   R11, 600(BP)
+	MOVL   R12, 48(BP)
+	MOVL   R12, 100(BP)
+	MOVL   R12, 160(BP)
+	MOVL   R12, 268(BP)
+	MOVL   R12, 328(BP)
+	MOVL   R12, 348(BP)
+	MOVL   R12, 444(BP)
+	MOVL   R12, 504(BP)
+	MOVL   R12, 556(BP)
+	MOVL   R12, 596(BP)
+	SHRQ   $0x20, R12
+	MOVL   R12, 64(BP)
+	MOVL   R12, 88(BP)
+	MOVL   R12, 188(BP)
+	MOVL   R12, 224(BP)
+	MOVL   R12, 272(BP)
+	MOVL   R12, 396(BP)
+	MOVL   R12, 440(BP)
+	MOVL   R12, 492(BP)
+	MOVL   R12, 548(BP)
+	MOVL   R12, 628(BP)
+	MOVL   R13, 52(BP)
+	MOVL   R13, 96(BP)
+	MOVL   R13, 176(BP)
+	MOVL   R13, 260(BP)
+	MOVL   R13, 284(BP)
+	MOVL   R13, 356(BP)
+	MOVL   R13, 428(BP)
+	MOVL   R13, 524(BP)
+	MOVL   R13, 572(BP)
+	MOVL   R13, 592(BP)
+	SHRQ   $0x20, R13
+	MOVL   R13, 68(BP)
+	MOVL   R13, 120(BP)
+	MOVL   R13, 144(BP)
+	MOVL   R13, 220(BP)
+	MOVL   R13, 308(BP)
+	MOVL   R13, 360(BP)
+	MOVL   R13, 460(BP)
+	MOVL   R13, 480(BP)
+	MOVL   R13, 536(BP)
+	MOVL   R13, 640(BP)
+	MOVL   R14, 56(BP)
+	MOVL   R14, 128(BP)
+	MOVL   R14, 148(BP)
+	MOVL   R14, 232(BP)
+	MOVL   R14, 324(BP)
+	MOVL   R14, 352(BP)
+	MOVL   R14, 400(BP)
+	MOVL   R14, 472(BP)
+	MOVL   R14, 560(BP)
+	MOVL   R14, 648(BP)
+	SHRQ   $0x20, R14
+	MOVL   R14, 72(BP)
+	MOVL   R14, 92(BP)
+	MOVL   R14, 172(BP)
+	MOVL   R14, 216(BP)
+	MOVL   R14, 332(BP)
+	MOVL   R14, 384(BP)
+	MOVL   R14, 424(BP)
+	MOVL   R14, 464(BP)
+	MOVL   R14, 564(BP)
+	MOVL   R14, 636(BP)
+	MOVL   R15, 60(BP)
+	MOVL   R15, 80(BP)
+	MOVL   R15, 192(BP)
+	MOVL   R15, 236(BP)
+	MOVL   R15, 304(BP)
+	MOVL   R15, 392(BP)
+	MOVL   R15, 408(BP)
+	MOVL   R15, 484(BP)
+	MOVL   R15, 532(BP)
+	MOVL   R15, 644(BP)
+	SHRQ   $0x20, R15
+	MOVL   R15, 76(BP)
+	MOVL   R15, 104(BP)
+	MOVL   R15, 156(BP)
+	MOVL   R15, 252(BP)
+	MOVL   R15, 300(BP)
+	MOVL   R15, 376(BP)
+	MOVL   R15, 420(BP)
+	MOVL   R15, 500(BP)
+	MOVL   R15, 544(BP)
+	MOVL   R15, 624(BP)
+	PADDL  16(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  32(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  48(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  64(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  80(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  96(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  112(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  128(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  144(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  160(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  176(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  192(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  208(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  224(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  240(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  256(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  272(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  288(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  304(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  320(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  336(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  352(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  368(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  384(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  400(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  416(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  432(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  448(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  464(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  480(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  496(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  512(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  528(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  544(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  560(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  576(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PADDL  592(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  608(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  624(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  640(BP), X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PXOR   X4, X0
+	PXOR   X5, X1
+	PXOR   X6, X0
+	PXOR   X7, X1
+	LEAQ   64(SI), SI
+	SUBQ   $0x40, DX
+	JNE    loop
+	MOVO   X15, (BP)
+	MOVQ   (BP), R9
+	MOVQ   R9, (BX)
+	MOVOU  X0, (AX)
+	MOVOU  X1, 16(AX)
 	RET
 
 // func hashBlocksSSE4(h *[8]uint32, c *[2]uint32, flag uint32, blocks []byte)
-TEXT ·hashBlocksSSE4(SB), 0, $32-48 // frame = 16 + 16 byte alignment
-	HASH_BLOCKS(h+0(FP), c+8(FP), flag+16(FP), blocks_base+24(FP), blocks_len+32(FP), BLAKE2s_SSE4)
+// Requires: SSE2, SSE4.1, SSSE3
+TEXT ·hashBlocksSSE4(SB), $32-48
+	MOVQ  h+0(FP), AX
+	MOVQ  c+8(FP), BX
+	MOVL  flag+16(FP), CX
+	MOVQ  blocks_base+24(FP), SI
+	MOVQ  blocks_len+32(FP), DX
+	MOVQ  SP, BP
+	ADDQ  $0x0f, BP
+	ANDQ  $-16, BP
+	MOVQ  (BX), R9
+	MOVQ  R9, (BP)
+	MOVQ  CX, 8(BP)
+	MOVOU (AX), X0
+	MOVOU 16(AX), X1
+	MOVOU iv0<>+0(SB), X2
+	MOVOU iv1<>+0(SB), X3
+	MOVOU counter<>+0(SB), X12
+	MOVOU rol16<>+0(SB), X13
+	MOVOU rol8<>+0(SB), X14
+	MOVO  (BP), X15
+
+loop:
+	MOVO   X0, X4
+	MOVO   X1, X5
+	MOVO   X2, X6
+	MOVO   X3, X7
+	PADDQ  X12, X15
+	PXOR   X15, X7
+	MOVL   (SI), X8
+	PINSRD $0x01, 8(SI), X8
+	PINSRD $0x02, 16(SI), X8
+	PINSRD $0x03, 24(SI), X8
+	MOVL   4(SI), X9
+	PINSRD $0x01, 12(SI), X9
+	PINSRD $0x02, 20(SI), X9
+	PINSRD $0x03, 28(SI), X9
+	MOVL   32(SI), X10
+	PINSRD $0x01, 40(SI), X10
+	PINSRD $0x02, 48(SI), X10
+	PINSRD $0x03, 56(SI), X10
+	MOVL   36(SI), X11
+	PINSRD $0x01, 44(SI), X11
+	PINSRD $0x02, 52(SI), X11
+	PINSRD $0x03, 60(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   56(SI), X8
+	PINSRD $0x01, 16(SI), X8
+	PINSRD $0x02, 36(SI), X8
+	PINSRD $0x03, 52(SI), X8
+	MOVL   40(SI), X9
+	PINSRD $0x01, 32(SI), X9
+	PINSRD $0x02, 60(SI), X9
+	PINSRD $0x03, 24(SI), X9
+	MOVL   4(SI), X10
+	PINSRD $0x01, (SI), X10
+	PINSRD $0x02, 44(SI), X10
+	PINSRD $0x03, 20(SI), X10
+	MOVL   48(SI), X11
+	PINSRD $0x01, 8(SI), X11
+	PINSRD $0x02, 28(SI), X11
+	PINSRD $0x03, 12(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   44(SI), X8
+	PINSRD $0x01, 48(SI), X8
+	PINSRD $0x02, 20(SI), X8
+	PINSRD $0x03, 60(SI), X8
+	MOVL   32(SI), X9
+	PINSRD $0x01, (SI), X9
+	PINSRD $0x02, 8(SI), X9
+	PINSRD $0x03, 52(SI), X9
+	MOVL   40(SI), X10
+	PINSRD $0x01, 12(SI), X10
+	PINSRD $0x02, 28(SI), X10
+	PINSRD $0x03, 36(SI), X10
+	MOVL   56(SI), X11
+	PINSRD $0x01, 24(SI), X11
+	PINSRD $0x02, 4(SI), X11
+	PINSRD $0x03, 16(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   28(SI), X8
+	PINSRD $0x01, 12(SI), X8
+	PINSRD $0x02, 52(SI), X8
+	PINSRD $0x03, 44(SI), X8
+	MOVL   36(SI), X9
+	PINSRD $0x01, 4(SI), X9
+	PINSRD $0x02, 48(SI), X9
+	PINSRD $0x03, 56(SI), X9
+	MOVL   8(SI), X10
+	PINSRD $0x01, 20(SI), X10
+	PINSRD $0x02, 16(SI), X10
+	PINSRD $0x03, 60(SI), X10
+	MOVL   24(SI), X11
+	PINSRD $0x01, 40(SI), X11
+	PINSRD $0x02, (SI), X11
+	PINSRD $0x03, 32(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   36(SI), X8
+	PINSRD $0x01, 20(SI), X8
+	PINSRD $0x02, 8(SI), X8
+	PINSRD $0x03, 40(SI), X8
+	MOVL   (SI), X9
+	PINSRD $0x01, 28(SI), X9
+	PINSRD $0x02, 16(SI), X9
+	PINSRD $0x03, 60(SI), X9
+	MOVL   56(SI), X10
+	PINSRD $0x01, 44(SI), X10
+	PINSRD $0x02, 24(SI), X10
+	PINSRD $0x03, 12(SI), X10
+	MOVL   4(SI), X11
+	PINSRD $0x01, 48(SI), X11
+	PINSRD $0x02, 32(SI), X11
+	PINSRD $0x03, 52(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   8(SI), X8
+	PINSRD $0x01, 24(SI), X8
+	PINSRD $0x02, (SI), X8
+	PINSRD $0x03, 32(SI), X8
+	MOVL   48(SI), X9
+	PINSRD $0x01, 40(SI), X9
+	PINSRD $0x02, 44(SI), X9
+	PINSRD $0x03, 12(SI), X9
+	MOVL   16(SI), X10
+	PINSRD $0x01, 28(SI), X10
+	PINSRD $0x02, 60(SI), X10
+	PINSRD $0x03, 4(SI), X10
+	MOVL   52(SI), X11
+	PINSRD $0x01, 20(SI), X11
+	PINSRD $0x02, 56(SI), X11
+	PINSRD $0x03, 36(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   48(SI), X8
+	PINSRD $0x01, 4(SI), X8
+	PINSRD $0x02, 56(SI), X8
+	PINSRD $0x03, 16(SI), X8
+	MOVL   20(SI), X9
+	PINSRD $0x01, 60(SI), X9
+	PINSRD $0x02, 52(SI), X9
+	PINSRD $0x03, 40(SI), X9
+	MOVL   (SI), X10
+	PINSRD $0x01, 24(SI), X10
+	PINSRD $0x02, 36(SI), X10
+	PINSRD $0x03, 32(SI), X10
+	MOVL   28(SI), X11
+	PINSRD $0x01, 12(SI), X11
+	PINSRD $0x02, 8(SI), X11
+	PINSRD $0x03, 44(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   52(SI), X8
+	PINSRD $0x01, 28(SI), X8
+	PINSRD $0x02, 48(SI), X8
+	PINSRD $0x03, 12(SI), X8
+	MOVL   44(SI), X9
+	PINSRD $0x01, 56(SI), X9
+	PINSRD $0x02, 4(SI), X9
+	PINSRD $0x03, 36(SI), X9
+	MOVL   20(SI), X10
+	PINSRD $0x01, 60(SI), X10
+	PINSRD $0x02, 32(SI), X10
+	PINSRD $0x03, 8(SI), X10
+	MOVL   (SI), X11
+	PINSRD $0x01, 16(SI), X11
+	PINSRD $0x02, 24(SI), X11
+	PINSRD $0x03, 40(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   24(SI), X8
+	PINSRD $0x01, 56(SI), X8
+	PINSRD $0x02, 44(SI), X8
+	PINSRD $0x03, (SI), X8
+	MOVL   60(SI), X9
+	PINSRD $0x01, 36(SI), X9
+	PINSRD $0x02, 12(SI), X9
+	PINSRD $0x03, 32(SI), X9
+	MOVL   48(SI), X10
+	PINSRD $0x01, 52(SI), X10
+	PINSRD $0x02, 4(SI), X10
+	PINSRD $0x03, 40(SI), X10
+	MOVL   8(SI), X11
+	PINSRD $0x01, 28(SI), X11
+	PINSRD $0x02, 16(SI), X11
+	PINSRD $0x03, 20(SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	MOVL   40(SI), X8
+	PINSRD $0x01, 32(SI), X8
+	PINSRD $0x02, 28(SI), X8
+	PINSRD $0x03, 4(SI), X8
+	MOVL   8(SI), X9
+	PINSRD $0x01, 16(SI), X9
+	PINSRD $0x02, 24(SI), X9
+	PINSRD $0x03, 20(SI), X9
+	MOVL   60(SI), X10
+	PINSRD $0x01, 36(SI), X10
+	PINSRD $0x02, 12(SI), X10
+	PINSRD $0x03, 52(SI), X10
+	MOVL   44(SI), X11
+	PINSRD $0x01, 56(SI), X11
+	PINSRD $0x02, 48(SI), X11
+	PINSRD $0x03, (SI), X11
+	PADDL  X8, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X9, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X5, X5
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X7, X7
+	PADDL  X10, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X13, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x14, X8
+	PSRLL  $0x0c, X5
+	PXOR   X8, X5
+	PADDL  X11, X4
+	PADDL  X5, X4
+	PXOR   X4, X7
+	PSHUFB X14, X7
+	PADDL  X7, X6
+	PXOR   X6, X5
+	MOVO   X5, X8
+	PSLLL  $0x19, X8
+	PSRLL  $0x07, X5
+	PXOR   X8, X5
+	PSHUFL $0x39, X7, X7
+	PSHUFL $0x4e, X6, X6
+	PSHUFL $0x93, X5, X5
+	PXOR   X4, X0
+	PXOR   X5, X1
+	PXOR   X6, X0
+	PXOR   X7, X1
+	LEAQ   64(SI), SI
+	SUBQ   $0x40, DX
+	JNE    loop
+	MOVO   X15, (BP)
+	MOVQ   (BP), R9
+	MOVQ   R9, (BX)
+	MOVOU  X0, (AX)
+	MOVOU  X1, 16(AX)
 	RET