From 38a0b5da75b2e8dab0b735296f86d9605e5f92cf Mon Sep 17 00:00:00 2001
From: Garrett Bodley <garrett.bodley@gmail.com>
Date: Mon, 22 Jul 2024 16:52:53 -0400
Subject: [PATCH] argon2: Avo port of blamka_amd64.s

This implementation utilizes the same registers found in the reference
implementation, aiming to produce a minimal semantic diff between the
Avo-generated output and the original hand-written assembly.

To verify the Avo implementation, the reference and Avo-generated
assembly files are fed to `go tool asm`, capturing the debug output into
corresponding temp files. The debug output contains supplementary
metadata (line numbers, instruction offsets, and source file references)
that must be removed in order to obtain a semantic diff of the two
files. This is accomplished via a small utility script written in awk.

Commands used to verify Avo output:

GOROOT=$(go env GOROOT)
ASM_PATH="argon2/blamka_amd64.s"
REFERENCE="b2d3a6a4b4d36521cd7f653879cf6981e7c5c340"

go tool asm -o /dev/null -I "$GOROOT"/src/runtime -debug \
  <(git cat-file -p "$REFERENCE:$ASM_PATH") \
  > /tmp/reference.s

go tool asm -o /dev/null -I "$GOROOT"/src/runtime -debug \
  "$ASM_PATH" \
  > /tmp/avo.s

normalize(){
  awk '{
    $1=$2=$3="";
    print substr($0,4)
  }'
}

diff <(normalize < /tmp/reference.s) <(normalize < /tmp/avo.s)

Change-Id: I3567eb80ef80dff248225f17470122c0a4e6951e
Reviewed-on: https://go-review.googlesource.com/c/crypto/+/600315
Reviewed-by: Filippo Valsorda <filippo@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Roland Shoemaker <roland@golang.org>
---
 argon2/_asm/blamka_amd64.go |  287 ++++
 argon2/_asm/go.mod          |   15 +
 argon2/_asm/go.sum          |   12 +
 argon2/blamka_amd64.s       | 2972 ++++++++++++++++++++++++++++++++---
 4 files changed, 3074 insertions(+), 212 deletions(-)
 create mode 100644 argon2/_asm/blamka_amd64.go
 create mode 100644 argon2/_asm/go.mod
 create mode 100644 argon2/_asm/go.sum

diff --git a/argon2/_asm/blamka_amd64.go b/argon2/_asm/blamka_amd64.go
new file mode 100644
index 0000000000..17a1e7629a
--- /dev/null
+++ b/argon2/_asm/blamka_amd64.go
@@ -0,0 +1,287 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+	. "github.com/mmcloughlin/avo/reg"
+	_ "golang.org/x/crypto/argon2"
+)
+
+//go:generate go run . -out ../blamka_amd64.s -pkg argon2
+
+func main() {
+	Package("golang.org/x/crypto/argon2")
+	ConstraintExpr("amd64,gc,!purego")
+
+	blamkaSSE4()
+	mixBlocksSSE2()
+	xorBlocksSSE2()
+	Generate()
+}
+
+func blamkaSSE4() {
+	Implement("blamkaSSE4")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("b"), RAX)
+
+	c40 := c40_DATA()
+	c48 := c48_DATA()
+
+	MOVOU(c40, X10)
+	MOVOU(c48, X11)
+
+	BLAMKA_ROUND_0(AX, 0, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 16, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 32, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 48, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 64, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 80, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 96, X8, X9, X10, X11)
+	BLAMKA_ROUND_0(AX, 112, X8, X9, X10, X11)
+
+	BLAMKA_ROUND_1(AX, 0, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 2, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 4, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 6, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 8, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 10, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 12, X8, X9, X10, X11)
+	BLAMKA_ROUND_1(AX, 14, X8, X9, X10, X11)
+	RET()
+}
+
+func mixBlocksSSE2() {
+	Implement("mixBlocksSSE2")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("out"), RDX)
+	Load(Param("a"), RAX)
+	Load(Param("b"), RBX)
+	Load(Param("c"), RCX)
+	MOVQ(U32(128), RDI)
+
+	Label("loop")
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: BX}.Offset(0), X1)
+	MOVOU(Mem{Base: CX}.Offset(0), X2)
+	PXOR(X1, X0)
+	PXOR(X2, X0)
+	MOVOU(X0, Mem{Base: DX}.Offset(0))
+	ADDQ(Imm(16), RAX)
+	ADDQ(Imm(16), RBX)
+	ADDQ(Imm(16), RCX)
+	ADDQ(Imm(16), RDX)
+	SUBQ(Imm(2), RDI)
+	JA(LabelRef("loop"))
+	RET()
+}
+
+func xorBlocksSSE2() {
+	Implement("xorBlocksSSE2")
+	Attributes(NOSPLIT)
+	AllocLocal(0)
+
+	Load(Param("out"), RDX)
+	Load(Param("a"), RAX)
+	Load(Param("b"), RBX)
+	Load(Param("c"), RCX)
+	MOVQ(U32(128), RDI)
+
+	Label("loop")
+	MOVOU(Mem{Base: AX}.Offset(0), X0)
+	MOVOU(Mem{Base: BX}.Offset(0), X1)
+	MOVOU(Mem{Base: CX}.Offset(0), X2)
+	MOVOU(Mem{Base: DX}.Offset(0), X3)
+	PXOR(X1, X0)
+	PXOR(X2, X0)
+	PXOR(X3, X0)
+	MOVOU(X0, Mem{Base: DX}.Offset(0))
+	ADDQ(Imm(16), RAX)
+	ADDQ(Imm(16), RBX)
+	ADDQ(Imm(16), RCX)
+	ADDQ(Imm(16), RDX)
+	SUBQ(Imm(2), RDI)
+	JA(LabelRef("loop"))
+	RET()
+}
+
+func SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v6, t1)
+	PUNPCKLQDQ(v6, t2)
+	PUNPCKHQDQ(v7, v6)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(v7, t2)
+	MOVO(t1, v7)
+	MOVO(v2, t1)
+	PUNPCKHQDQ(t2, v7)
+	PUNPCKLQDQ(v3, t2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v3)
+}
+
+func SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2 VecPhysical) {
+	MOVO(v4, t1)
+	MOVO(v5, v4)
+	MOVO(t1, v5)
+	MOVO(v2, t1)
+	PUNPCKLQDQ(v2, t2)
+	PUNPCKHQDQ(v3, v2)
+	PUNPCKHQDQ(t2, v2)
+	PUNPCKLQDQ(v3, t2)
+	MOVO(t1, v3)
+	MOVO(v6, t1)
+	PUNPCKHQDQ(t2, v3)
+	PUNPCKLQDQ(v7, t2)
+	PUNPCKHQDQ(t2, v6)
+	PUNPCKLQDQ(t1, t2)
+	PUNPCKHQDQ(t2, v7)
+}
+
+func HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, t0, c40, c48 VecPhysical) {
+	MOVO(v0, t0)
+	PMULULQ(v2, t0)
+	PADDQ(v2, v0)
+	PADDQ(t0, v0)
+	PADDQ(t0, v0)
+	PXOR(v0, v6)
+	PSHUFD(Imm(0xB1), v6, v6)
+	MOVO(v4, t0)
+	PMULULQ(v6, t0)
+	PADDQ(v6, v4)
+	PADDQ(t0, v4)
+	PADDQ(t0, v4)
+	PXOR(v4, v2)
+	PSHUFB(c40, v2)
+	MOVO(v0, t0)
+	PMULULQ(v2, t0)
+	PADDQ(v2, v0)
+	PADDQ(t0, v0)
+	PADDQ(t0, v0)
+	PXOR(v0, v6)
+	PSHUFB(c48, v6)
+	MOVO(v4, t0)
+	PMULULQ(v6, t0)
+	PADDQ(v6, v4)
+	PADDQ(t0, v4)
+	PADDQ(t0, v4)
+	PXOR(v4, v2)
+	MOVO(v2, t0)
+	PADDQ(v2, t0)
+	PSRLQ(Imm(63), v2)
+	PXOR(t0, v2)
+	MOVO(v1, t0)
+	PMULULQ(v3, t0)
+	PADDQ(v3, v1)
+	PADDQ(t0, v1)
+	PADDQ(t0, v1)
+	PXOR(v1, v7)
+	PSHUFD(Imm(0xB1), v7, v7)
+	MOVO(v5, t0)
+	PMULULQ(v7, t0)
+	PADDQ(v7, v5)
+	PADDQ(t0, v5)
+	PADDQ(t0, v5)
+	PXOR(v5, v3)
+	PSHUFB(c40, v3)
+	MOVO(v1, t0)
+	PMULULQ(v3, t0)
+	PADDQ(v3, v1)
+	PADDQ(t0, v1)
+	PADDQ(t0, v1)
+	PXOR(v1, v7)
+	PSHUFB(c48, v7)
+	MOVO(v5, t0)
+	PMULULQ(v7, t0)
+	PADDQ(v7, v5)
+	PADDQ(t0, v5)
+	PADDQ(t0, v5)
+	PXOR(v5, v3)
+	MOVO(v3, t0)
+	PADDQ(v3, t0)
+	PSRLQ(Imm(63), v3)
+	PXOR(t0, v3)
+}
+
+func LOAD_MSG_0(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(Mem{Base: block}.Offset(8*(off+(i*2))), r)
+	}
+}
+
+func STORE_MSG_0(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(r, Mem{Base: block}.Offset(8*(off+(i*2))))
+	}
+}
+
+func LOAD_MSG_1(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(Mem{Base: block}.Offset(8*off+i*16*8), r)
+	}
+}
+
+func STORE_MSG_1(block GPPhysical, off int) {
+	var registers = []VecPhysical{X0, X1, X2, X3, X4, X5, X6, X7}
+	for i, r := range registers {
+		MOVOU(r, Mem{Base: block}.Offset(8*off+i*16*8))
+	}
+}
+
+func BLAMKA_ROUND_0(block GPPhysical, off int, t0, t1, c40, c48 VecPhysical) {
+	LOAD_MSG_0(block, off)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1)
+	STORE_MSG_0(block, off)
+}
+
+func BLAMKA_ROUND_1(block GPPhysical, off int, t0, t1, c40, c48 VecPhysical) {
+	LOAD_MSG_1(block, off)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1)
+	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48)
+	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1)
+	STORE_MSG_1(block, off)
+}
+
+// ##------------------DATA SECTION-------------------##
+
+var c40_DATA_ptr, c48_DATA_ptr *Mem
+
+func c40_DATA() Mem {
+	if c40_DATA_ptr != nil {
+		return *c40_DATA_ptr
+	}
+
+	c40_DATA := GLOBL("·c40", NOPTR|RODATA)
+	c40_DATA_ptr = &c40_DATA
+	DATA(0x00, U64(0x0201000706050403))
+	DATA(0x08, U64(0x0a09080f0e0d0c0b))
+	return c40_DATA
+}
+func c48_DATA() Mem {
+	if c48_DATA_ptr != nil {
+		return *c48_DATA_ptr
+	}
+
+	c48_DATA := GLOBL("·c48", NOPTR|RODATA)
+	c48_DATA_ptr = &c48_DATA
+	DATA(0x00, U64(0x0100070605040302))
+	DATA(0x08, U64(0x09080f0e0d0c0b0a))
+	return c48_DATA
+}
diff --git a/argon2/_asm/go.mod b/argon2/_asm/go.mod
new file mode 100644
index 0000000000..41a536dd77
--- /dev/null
+++ b/argon2/_asm/go.mod
@@ -0,0 +1,15 @@
+module argon2/_asm
+
+go 1.23
+
+require (
+	github.com/mmcloughlin/avo v0.6.0
+	golang.org/x/crypto v0.26.0
+)
+
+require (
+	golang.org/x/mod v0.20.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.24.0 // indirect
+	golang.org/x/tools v0.24.0 // indirect
+)
diff --git a/argon2/_asm/go.sum b/argon2/_asm/go.sum
new file mode 100644
index 0000000000..62ea9dfb70
--- /dev/null
+++ b/argon2/_asm/go.sum
@@ -0,0 +1,12 @@
+github.com/mmcloughlin/avo v0.6.0 h1:QH6FU8SKoTLaVs80GA8TJuLNkUYl4VokHKlPhVDg4YY=
+github.com/mmcloughlin/avo v0.6.0/go.mod h1:8CoAGaCSYXtCPR+8y18Y9aB/kxb8JSS6FRI7mSkvD+8=
+golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw=
+golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54=
+golang.org/x/mod v0.20.0 h1:utOm6MM3R3dnawAiJgn0y+xvuYRsm1RKM/4giyfDgV0=
+golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
+golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
+golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24=
+golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ=
diff --git a/argon2/blamka_amd64.s b/argon2/blamka_amd64.s
index 6713accac0..c3895478ed 100644
--- a/argon2/blamka_amd64.s
+++ b/argon2/blamka_amd64.s
@@ -1,243 +1,2791 @@
-// Copyright 2017 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
+// Code generated by command: go run blamka_amd64.go -out ../blamka_amd64.s -pkg argon2. DO NOT EDIT.
 
 //go:build amd64 && gc && !purego
 
 #include "textflag.h"
 
-DATA ·c40<>+0x00(SB)/8, $0x0201000706050403
-DATA ·c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b
-GLOBL ·c40<>(SB), (NOPTR+RODATA), $16
-
-DATA ·c48<>+0x00(SB)/8, $0x0100070605040302
-DATA ·c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a
-GLOBL ·c48<>(SB), (NOPTR+RODATA), $16
-
-#define SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2) \
-	MOVO       v4, t1; \
-	MOVO       v5, v4; \
-	MOVO       t1, v5; \
-	MOVO       v6, t1; \
-	PUNPCKLQDQ v6, t2; \
-	PUNPCKHQDQ v7, v6; \
-	PUNPCKHQDQ t2, v6; \
-	PUNPCKLQDQ v7, t2; \
-	MOVO       t1, v7; \
-	MOVO       v2, t1; \
-	PUNPCKHQDQ t2, v7; \
-	PUNPCKLQDQ v3, t2; \
-	PUNPCKHQDQ t2, v2; \
-	PUNPCKLQDQ t1, t2; \
-	PUNPCKHQDQ t2, v3
-
-#define SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2) \
-	MOVO       v4, t1; \
-	MOVO       v5, v4; \
-	MOVO       t1, v5; \
-	MOVO       v2, t1; \
-	PUNPCKLQDQ v2, t2; \
-	PUNPCKHQDQ v3, v2; \
-	PUNPCKHQDQ t2, v2; \
-	PUNPCKLQDQ v3, t2; \
-	MOVO       t1, v3; \
-	MOVO       v6, t1; \
-	PUNPCKHQDQ t2, v3; \
-	PUNPCKLQDQ v7, t2; \
-	PUNPCKHQDQ t2, v6; \
-	PUNPCKLQDQ t1, t2; \
-	PUNPCKHQDQ t2, v7
-
-#define HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, t0, c40, c48) \
-	MOVO    v0, t0;        \
-	PMULULQ v2, t0;        \
-	PADDQ   v2, v0;        \
-	PADDQ   t0, v0;        \
-	PADDQ   t0, v0;        \
-	PXOR    v0, v6;        \
-	PSHUFD  $0xB1, v6, v6; \
-	MOVO    v4, t0;        \
-	PMULULQ v6, t0;        \
-	PADDQ   v6, v4;        \
-	PADDQ   t0, v4;        \
-	PADDQ   t0, v4;        \
-	PXOR    v4, v2;        \
-	PSHUFB  c40, v2;       \
-	MOVO    v0, t0;        \
-	PMULULQ v2, t0;        \
-	PADDQ   v2, v0;        \
-	PADDQ   t0, v0;        \
-	PADDQ   t0, v0;        \
-	PXOR    v0, v6;        \
-	PSHUFB  c48, v6;       \
-	MOVO    v4, t0;        \
-	PMULULQ v6, t0;        \
-	PADDQ   v6, v4;        \
-	PADDQ   t0, v4;        \
-	PADDQ   t0, v4;        \
-	PXOR    v4, v2;        \
-	MOVO    v2, t0;        \
-	PADDQ   v2, t0;        \
-	PSRLQ   $63, v2;       \
-	PXOR    t0, v2;        \
-	MOVO    v1, t0;        \
-	PMULULQ v3, t0;        \
-	PADDQ   v3, v1;        \
-	PADDQ   t0, v1;        \
-	PADDQ   t0, v1;        \
-	PXOR    v1, v7;        \
-	PSHUFD  $0xB1, v7, v7; \
-	MOVO    v5, t0;        \
-	PMULULQ v7, t0;        \
-	PADDQ   v7, v5;        \
-	PADDQ   t0, v5;        \
-	PADDQ   t0, v5;        \
-	PXOR    v5, v3;        \
-	PSHUFB  c40, v3;       \
-	MOVO    v1, t0;        \
-	PMULULQ v3, t0;        \
-	PADDQ   v3, v1;        \
-	PADDQ   t0, v1;        \
-	PADDQ   t0, v1;        \
-	PXOR    v1, v7;        \
-	PSHUFB  c48, v7;       \
-	MOVO    v5, t0;        \
-	PMULULQ v7, t0;        \
-	PADDQ   v7, v5;        \
-	PADDQ   t0, v5;        \
-	PADDQ   t0, v5;        \
-	PXOR    v5, v3;        \
-	MOVO    v3, t0;        \
-	PADDQ   v3, t0;        \
-	PSRLQ   $63, v3;       \
-	PXOR    t0, v3
-
-#define LOAD_MSG_0(block, off) \
-	MOVOU 8*(off+0)(block), X0;  \
-	MOVOU 8*(off+2)(block), X1;  \
-	MOVOU 8*(off+4)(block), X2;  \
-	MOVOU 8*(off+6)(block), X3;  \
-	MOVOU 8*(off+8)(block), X4;  \
-	MOVOU 8*(off+10)(block), X5; \
-	MOVOU 8*(off+12)(block), X6; \
-	MOVOU 8*(off+14)(block), X7
-
-#define STORE_MSG_0(block, off) \
-	MOVOU X0, 8*(off+0)(block);  \
-	MOVOU X1, 8*(off+2)(block);  \
-	MOVOU X2, 8*(off+4)(block);  \
-	MOVOU X3, 8*(off+6)(block);  \
-	MOVOU X4, 8*(off+8)(block);  \
-	MOVOU X5, 8*(off+10)(block); \
-	MOVOU X6, 8*(off+12)(block); \
-	MOVOU X7, 8*(off+14)(block)
-
-#define LOAD_MSG_1(block, off) \
-	MOVOU 8*off+0*8(block), X0;  \
-	MOVOU 8*off+16*8(block), X1; \
-	MOVOU 8*off+32*8(block), X2; \
-	MOVOU 8*off+48*8(block), X3; \
-	MOVOU 8*off+64*8(block), X4; \
-	MOVOU 8*off+80*8(block), X5; \
-	MOVOU 8*off+96*8(block), X6; \
-	MOVOU 8*off+112*8(block), X7
-
-#define STORE_MSG_1(block, off) \
-	MOVOU X0, 8*off+0*8(block);  \
-	MOVOU X1, 8*off+16*8(block); \
-	MOVOU X2, 8*off+32*8(block); \
-	MOVOU X3, 8*off+48*8(block); \
-	MOVOU X4, 8*off+64*8(block); \
-	MOVOU X5, 8*off+80*8(block); \
-	MOVOU X6, 8*off+96*8(block); \
-	MOVOU X7, 8*off+112*8(block)
-
-#define BLAMKA_ROUND_0(block, off, t0, t1, c40, c48) \
-	LOAD_MSG_0(block, off);                                   \
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \
-	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1);                  \
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1);              \
-	STORE_MSG_0(block, off)
-
-#define BLAMKA_ROUND_1(block, off, t0, t1, c40, c48) \
-	LOAD_MSG_1(block, off);                                   \
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \
-	SHUFFLE(X2, X3, X4, X5, X6, X7, t0, t1);                  \
-	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, t0, c40, c48); \
-	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, t0, t1);              \
-	STORE_MSG_1(block, off)
-
 // func blamkaSSE4(b *block)
-TEXT ·blamkaSSE4(SB), 4, $0-8
-	MOVQ b+0(FP), AX
-
-	MOVOU ·c40<>(SB), X10
-	MOVOU ·c48<>(SB), X11
+// Requires: SSE2, SSSE3
+TEXT ·blamkaSSE4(SB), NOSPLIT, $0-8
+	MOVQ       b+0(FP), AX
+	MOVOU      ·c40<>+0(SB), X10
+	MOVOU      ·c48<>+0(SB), X11
+	MOVOU      (AX), X0
+	MOVOU      16(AX), X1
+	MOVOU      32(AX), X2
+	MOVOU      48(AX), X3
+	MOVOU      64(AX), X4
+	MOVOU      80(AX), X5
+	MOVOU      96(AX), X6
+	MOVOU      112(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, (AX)
+	MOVOU      X1, 16(AX)
+	MOVOU      X2, 32(AX)
+	MOVOU      X3, 48(AX)
+	MOVOU      X4, 64(AX)
+	MOVOU      X5, 80(AX)
+	MOVOU      X6, 96(AX)
+	MOVOU      X7, 112(AX)
+	MOVOU      128(AX), X0
+	MOVOU      144(AX), X1
+	MOVOU      160(AX), X2
+	MOVOU      176(AX), X3
+	MOVOU      192(AX), X4
+	MOVOU      208(AX), X5
+	MOVOU      224(AX), X6
+	MOVOU      240(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 128(AX)
+	MOVOU      X1, 144(AX)
+	MOVOU      X2, 160(AX)
+	MOVOU      X3, 176(AX)
+	MOVOU      X4, 192(AX)
+	MOVOU      X5, 208(AX)
+	MOVOU      X6, 224(AX)
+	MOVOU      X7, 240(AX)
+	MOVOU      256(AX), X0
+	MOVOU      272(AX), X1
+	MOVOU      288(AX), X2
+	MOVOU      304(AX), X3
+	MOVOU      320(AX), X4
+	MOVOU      336(AX), X5
+	MOVOU      352(AX), X6
+	MOVOU      368(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 256(AX)
+	MOVOU      X1, 272(AX)
+	MOVOU      X2, 288(AX)
+	MOVOU      X3, 304(AX)
+	MOVOU      X4, 320(AX)
+	MOVOU      X5, 336(AX)
+	MOVOU      X6, 352(AX)
+	MOVOU      X7, 368(AX)
+	MOVOU      384(AX), X0
+	MOVOU      400(AX), X1
+	MOVOU      416(AX), X2
+	MOVOU      432(AX), X3
+	MOVOU      448(AX), X4
+	MOVOU      464(AX), X5
+	MOVOU      480(AX), X6
+	MOVOU      496(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 384(AX)
+	MOVOU      X1, 400(AX)
+	MOVOU      X2, 416(AX)
+	MOVOU      X3, 432(AX)
+	MOVOU      X4, 448(AX)
+	MOVOU      X5, 464(AX)
+	MOVOU      X6, 480(AX)
+	MOVOU      X7, 496(AX)
+	MOVOU      512(AX), X0
+	MOVOU      528(AX), X1
+	MOVOU      544(AX), X2
+	MOVOU      560(AX), X3
+	MOVOU      576(AX), X4
+	MOVOU      592(AX), X5
+	MOVOU      608(AX), X6
+	MOVOU      624(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 512(AX)
+	MOVOU      X1, 528(AX)
+	MOVOU      X2, 544(AX)
+	MOVOU      X3, 560(AX)
+	MOVOU      X4, 576(AX)
+	MOVOU      X5, 592(AX)
+	MOVOU      X6, 608(AX)
+	MOVOU      X7, 624(AX)
+	MOVOU      640(AX), X0
+	MOVOU      656(AX), X1
+	MOVOU      672(AX), X2
+	MOVOU      688(AX), X3
+	MOVOU      704(AX), X4
+	MOVOU      720(AX), X5
+	MOVOU      736(AX), X6
+	MOVOU      752(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 640(AX)
+	MOVOU      X1, 656(AX)
+	MOVOU      X2, 672(AX)
+	MOVOU      X3, 688(AX)
+	MOVOU      X4, 704(AX)
+	MOVOU      X5, 720(AX)
+	MOVOU      X6, 736(AX)
+	MOVOU      X7, 752(AX)
+	MOVOU      768(AX), X0
+	MOVOU      784(AX), X1
+	MOVOU      800(AX), X2
+	MOVOU      816(AX), X3
+	MOVOU      832(AX), X4
+	MOVOU      848(AX), X5
+	MOVOU      864(AX), X6
+	MOVOU      880(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 768(AX)
+	MOVOU      X1, 784(AX)
+	MOVOU      X2, 800(AX)
+	MOVOU      X3, 816(AX)
+	MOVOU      X4, 832(AX)
+	MOVOU      X5, 848(AX)
+	MOVOU      X6, 864(AX)
+	MOVOU      X7, 880(AX)
+	MOVOU      896(AX), X0
+	MOVOU      912(AX), X1
+	MOVOU      928(AX), X2
+	MOVOU      944(AX), X3
+	MOVOU      960(AX), X4
+	MOVOU      976(AX), X5
+	MOVOU      992(AX), X6
+	MOVOU      1008(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 896(AX)
+	MOVOU      X1, 912(AX)
+	MOVOU      X2, 928(AX)
+	MOVOU      X3, 944(AX)
+	MOVOU      X4, 960(AX)
+	MOVOU      X5, 976(AX)
+	MOVOU      X6, 992(AX)
+	MOVOU      X7, 1008(AX)
+	MOVOU      (AX), X0
+	MOVOU      128(AX), X1
+	MOVOU      256(AX), X2
+	MOVOU      384(AX), X3
+	MOVOU      512(AX), X4
+	MOVOU      640(AX), X5
+	MOVOU      768(AX), X6
+	MOVOU      896(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, (AX)
+	MOVOU      X1, 128(AX)
+	MOVOU      X2, 256(AX)
+	MOVOU      X3, 384(AX)
+	MOVOU      X4, 512(AX)
+	MOVOU      X5, 640(AX)
+	MOVOU      X6, 768(AX)
+	MOVOU      X7, 896(AX)
+	MOVOU      16(AX), X0
+	MOVOU      144(AX), X1
+	MOVOU      272(AX), X2
+	MOVOU      400(AX), X3
+	MOVOU      528(AX), X4
+	MOVOU      656(AX), X5
+	MOVOU      784(AX), X6
+	MOVOU      912(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 16(AX)
+	MOVOU      X1, 144(AX)
+	MOVOU      X2, 272(AX)
+	MOVOU      X3, 400(AX)
+	MOVOU      X4, 528(AX)
+	MOVOU      X5, 656(AX)
+	MOVOU      X6, 784(AX)
+	MOVOU      X7, 912(AX)
+	MOVOU      32(AX), X0
+	MOVOU      160(AX), X1
+	MOVOU      288(AX), X2
+	MOVOU      416(AX), X3
+	MOVOU      544(AX), X4
+	MOVOU      672(AX), X5
+	MOVOU      800(AX), X6
+	MOVOU      928(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 32(AX)
+	MOVOU      X1, 160(AX)
+	MOVOU      X2, 288(AX)
+	MOVOU      X3, 416(AX)
+	MOVOU      X4, 544(AX)
+	MOVOU      X5, 672(AX)
+	MOVOU      X6, 800(AX)
+	MOVOU      X7, 928(AX)
+	MOVOU      48(AX), X0
+	MOVOU      176(AX), X1
+	MOVOU      304(AX), X2
+	MOVOU      432(AX), X3
+	MOVOU      560(AX), X4
+	MOVOU      688(AX), X5
+	MOVOU      816(AX), X6
+	MOVOU      944(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 48(AX)
+	MOVOU      X1, 176(AX)
+	MOVOU      X2, 304(AX)
+	MOVOU      X3, 432(AX)
+	MOVOU      X4, 560(AX)
+	MOVOU      X5, 688(AX)
+	MOVOU      X6, 816(AX)
+	MOVOU      X7, 944(AX)
+	MOVOU      64(AX), X0
+	MOVOU      192(AX), X1
+	MOVOU      320(AX), X2
+	MOVOU      448(AX), X3
+	MOVOU      576(AX), X4
+	MOVOU      704(AX), X5
+	MOVOU      832(AX), X6
+	MOVOU      960(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 64(AX)
+	MOVOU      X1, 192(AX)
+	MOVOU      X2, 320(AX)
+	MOVOU      X3, 448(AX)
+	MOVOU      X4, 576(AX)
+	MOVOU      X5, 704(AX)
+	MOVOU      X6, 832(AX)
+	MOVOU      X7, 960(AX)
+	MOVOU      80(AX), X0
+	MOVOU      208(AX), X1
+	MOVOU      336(AX), X2
+	MOVOU      464(AX), X3
+	MOVOU      592(AX), X4
+	MOVOU      720(AX), X5
+	MOVOU      848(AX), X6
+	MOVOU      976(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 80(AX)
+	MOVOU      X1, 208(AX)
+	MOVOU      X2, 336(AX)
+	MOVOU      X3, 464(AX)
+	MOVOU      X4, 592(AX)
+	MOVOU      X5, 720(AX)
+	MOVOU      X6, 848(AX)
+	MOVOU      X7, 976(AX)
+	MOVOU      96(AX), X0
+	MOVOU      224(AX), X1
+	MOVOU      352(AX), X2
+	MOVOU      480(AX), X3
+	MOVOU      608(AX), X4
+	MOVOU      736(AX), X5
+	MOVOU      864(AX), X6
+	MOVOU      992(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 96(AX)
+	MOVOU      X1, 224(AX)
+	MOVOU      X2, 352(AX)
+	MOVOU      X3, 480(AX)
+	MOVOU      X4, 608(AX)
+	MOVOU      X5, 736(AX)
+	MOVOU      X6, 864(AX)
+	MOVOU      X7, 992(AX)
+	MOVOU      112(AX), X0
+	MOVOU      240(AX), X1
+	MOVOU      368(AX), X2
+	MOVOU      496(AX), X3
+	MOVOU      624(AX), X4
+	MOVOU      752(AX), X5
+	MOVOU      880(AX), X6
+	MOVOU      1008(AX), X7
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X6, X8
+	PUNPCKLQDQ X6, X9
+	PUNPCKHQDQ X7, X6
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X7, X9
+	MOVO       X8, X7
+	MOVO       X2, X8
+	PUNPCKHQDQ X9, X7
+	PUNPCKLQDQ X3, X9
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X3
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFD     $0xb1, X6, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	PSHUFB     X10, X2
+	MOVO       X0, X8
+	PMULULQ    X2, X8
+	PADDQ      X2, X0
+	PADDQ      X8, X0
+	PADDQ      X8, X0
+	PXOR       X0, X6
+	PSHUFB     X11, X6
+	MOVO       X4, X8
+	PMULULQ    X6, X8
+	PADDQ      X6, X4
+	PADDQ      X8, X4
+	PADDQ      X8, X4
+	PXOR       X4, X2
+	MOVO       X2, X8
+	PADDQ      X2, X8
+	PSRLQ      $0x3f, X2
+	PXOR       X8, X2
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFD     $0xb1, X7, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	PSHUFB     X10, X3
+	MOVO       X1, X8
+	PMULULQ    X3, X8
+	PADDQ      X3, X1
+	PADDQ      X8, X1
+	PADDQ      X8, X1
+	PXOR       X1, X7
+	PSHUFB     X11, X7
+	MOVO       X5, X8
+	PMULULQ    X7, X8
+	PADDQ      X7, X5
+	PADDQ      X8, X5
+	PADDQ      X8, X5
+	PXOR       X5, X3
+	MOVO       X3, X8
+	PADDQ      X3, X8
+	PSRLQ      $0x3f, X3
+	PXOR       X8, X3
+	MOVO       X4, X8
+	MOVO       X5, X4
+	MOVO       X8, X5
+	MOVO       X2, X8
+	PUNPCKLQDQ X2, X9
+	PUNPCKHQDQ X3, X2
+	PUNPCKHQDQ X9, X2
+	PUNPCKLQDQ X3, X9
+	MOVO       X8, X3
+	MOVO       X6, X8
+	PUNPCKHQDQ X9, X3
+	PUNPCKLQDQ X7, X9
+	PUNPCKHQDQ X9, X6
+	PUNPCKLQDQ X8, X9
+	PUNPCKHQDQ X9, X7
+	MOVOU      X0, 112(AX)
+	MOVOU      X1, 240(AX)
+	MOVOU      X2, 368(AX)
+	MOVOU      X3, 496(AX)
+	MOVOU      X4, 624(AX)
+	MOVOU      X5, 752(AX)
+	MOVOU      X6, 880(AX)
+	MOVOU      X7, 1008(AX)
+	RET
 
-	BLAMKA_ROUND_0(AX, 0, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 16, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 32, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 48, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 64, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 80, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 96, X8, X9, X10, X11)
-	BLAMKA_ROUND_0(AX, 112, X8, X9, X10, X11)
+DATA ·c40<>+0(SB)/8, $0x0201000706050403
+DATA ·c40<>+8(SB)/8, $0x0a09080f0e0d0c0b
+GLOBL ·c40<>(SB), RODATA|NOPTR, $16
 
-	BLAMKA_ROUND_1(AX, 0, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 2, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 4, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 6, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 8, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 10, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 12, X8, X9, X10, X11)
-	BLAMKA_ROUND_1(AX, 14, X8, X9, X10, X11)
-	RET
+DATA ·c48<>+0(SB)/8, $0x0100070605040302
+DATA ·c48<>+8(SB)/8, $0x09080f0e0d0c0b0a
+GLOBL ·c48<>(SB), RODATA|NOPTR, $16
 
-// func mixBlocksSSE2(out, a, b, c *block)
-TEXT ·mixBlocksSSE2(SB), 4, $0-32
+// func mixBlocksSSE2(out *block, a *block, b *block, c *block)
+// Requires: SSE2
+TEXT ·mixBlocksSSE2(SB), NOSPLIT, $0-32
 	MOVQ out+0(FP), DX
 	MOVQ a+8(FP), AX
 	MOVQ b+16(FP), BX
 	MOVQ c+24(FP), CX
-	MOVQ $128, DI
+	MOVQ $0x00000080, DI
 
 loop:
-	MOVOU 0(AX), X0
-	MOVOU 0(BX), X1
-	MOVOU 0(CX), X2
+	MOVOU (AX), X0
+	MOVOU (BX), X1
+	MOVOU (CX), X2
 	PXOR  X1, X0
 	PXOR  X2, X0
-	MOVOU X0, 0(DX)
-	ADDQ  $16, AX
-	ADDQ  $16, BX
-	ADDQ  $16, CX
-	ADDQ  $16, DX
-	SUBQ  $2, DI
+	MOVOU X0, (DX)
+	ADDQ  $0x10, AX
+	ADDQ  $0x10, BX
+	ADDQ  $0x10, CX
+	ADDQ  $0x10, DX
+	SUBQ  $0x02, DI
 	JA    loop
 	RET
 
-// func xorBlocksSSE2(out, a, b, c *block)
-TEXT ·xorBlocksSSE2(SB), 4, $0-32
+// func xorBlocksSSE2(out *block, a *block, b *block, c *block)
+// Requires: SSE2
+TEXT ·xorBlocksSSE2(SB), NOSPLIT, $0-32
 	MOVQ out+0(FP), DX
 	MOVQ a+8(FP), AX
 	MOVQ b+16(FP), BX
 	MOVQ c+24(FP), CX
-	MOVQ $128, DI
+	MOVQ $0x00000080, DI
 
 loop:
-	MOVOU 0(AX), X0
-	MOVOU 0(BX), X1
-	MOVOU 0(CX), X2
-	MOVOU 0(DX), X3
+	MOVOU (AX), X0
+	MOVOU (BX), X1
+	MOVOU (CX), X2
+	MOVOU (DX), X3
 	PXOR  X1, X0
 	PXOR  X2, X0
 	PXOR  X3, X0
-	MOVOU X0, 0(DX)
-	ADDQ  $16, AX
-	ADDQ  $16, BX
-	ADDQ  $16, CX
-	ADDQ  $16, DX
-	SUBQ  $2, DI
+	MOVOU X0, (DX)
+	ADDQ  $0x10, AX
+	ADDQ  $0x10, BX
+	ADDQ  $0x10, CX
+	ADDQ  $0x10, DX
+	SUBQ  $0x02, DI
 	JA    loop
 	RET