From 00a3896dc57664aeefcd41da8acd12081bd91cda Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 5 Feb 2024 15:33:47 +0100 Subject: [PATCH] crypto/bls12381: update kilic library --- core/vm/contracts.go | 122 +- core/vm/contracts_test.go | 4 +- crypto/bls12381/arithmetic_decl.go | 197 +- crypto/bls12381/arithmetic_fallback.go | 511 +-- crypto/bls12381/bls12_381.go | 277 +- crypto/bls12381/bls12_381_test.go | 58 +- crypto/bls12381/field_element.go | 174 +- crypto/bls12381/field_element_test.go | 47 +- crypto/bls12381/fp.go | 255 +- crypto/bls12381/fp12.go | 354 ++- crypto/bls12381/fp2.go | 292 +- crypto/bls12381/fp2_arithmetic_x86.s | 4025 ++++++++++++++++++++++++ crypto/bls12381/fp6.go | 625 ++-- crypto/bls12381/fp_arithmetic_x86.s | 3894 +++++++++++++++++++++++ crypto/bls12381/fp_fallback.go | 1371 ++++++++ crypto/bls12381/fp_test.go | 1821 ++++++++--- crypto/bls12381/fr.go | 455 +++ crypto/bls12381/fr_arithmetic_x86.s | 1411 +++++++++ crypto/bls12381/fr_fallback.go | 383 +++ crypto/bls12381/fr_test.go | 417 +++ crypto/bls12381/g1.go | 810 +++-- crypto/bls12381/g1_test.go | 594 +++- crypto/bls12381/g2.go | 862 +++-- crypto/bls12381/g2_test.go | 607 +++- crypto/bls12381/glv.go | 200 ++ crypto/bls12381/glv_test.go | 134 + crypto/bls12381/gt.go | 28 +- crypto/bls12381/hash_to_field.go | 70 + crypto/bls12381/isogeny.go | 164 +- crypto/bls12381/pairing.go | 374 ++- crypto/bls12381/pairing_test.go | 128 +- crypto/bls12381/swu.go | 30 +- crypto/bls12381/utils.go | 40 +- crypto/bls12381/wnaf.go | 110 + crypto/bls12381/wnaf_test.go | 67 + 35 files changed, 18210 insertions(+), 2701 deletions(-) create mode 100644 crypto/bls12381/fp2_arithmetic_x86.s create mode 100644 crypto/bls12381/fp_arithmetic_x86.s create mode 100644 crypto/bls12381/fp_fallback.go create mode 100644 crypto/bls12381/fr.go create mode 100644 crypto/bls12381/fr_arithmetic_x86.s create mode 100644 crypto/bls12381/fr_fallback.go create mode 100644 crypto/bls12381/fr_test.go create mode 100644 crypto/bls12381/glv.go create mode 100644 crypto/bls12381/glv_test.go create mode 100644 crypto/bls12381/hash_to_field.go create mode 100644 crypto/bls12381/wnaf.go create mode 100644 crypto/bls12381/wnaf_test.go diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 2991437608..11ac79d65c 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -679,11 +679,11 @@ func (c *bls12381G1Add) Run(input []byte) ([]byte, error) { g := bls12381.NewG1() // Decode G1 point p_0 - if p0, err = g.DecodePoint(input[:128]); err != nil { + if p0, err = decodePointG1(g, input[:128]); err != nil { return nil, err } // Decode G1 point p_1 - if p1, err = g.DecodePoint(input[128:]); err != nil { + if p1, err = decodePointG1(g, input[128:]); err != nil { return nil, err } @@ -692,7 +692,7 @@ func (c *bls12381G1Add) Run(input []byte) ([]byte, error) { g.Add(r, p0, p1) // Encode the G1 point result into 128 bytes - return g.EncodePoint(r), nil + return encodePointG1(g, r), nil } // bls12381G1Mul implements EIP-2537 G1Mul precompile. @@ -717,7 +717,7 @@ func (c *bls12381G1Mul) Run(input []byte) ([]byte, error) { g := bls12381.NewG1() // Decode G1 point - if p0, err = g.DecodePoint(input[:128]); err != nil { + if p0, err = decodePointG1(g, input[:128]); err != nil { return nil, err } // Decode scalar value @@ -725,10 +725,10 @@ func (c *bls12381G1Mul) Run(input []byte) ([]byte, error) { // Compute r = e * p_0 r := g.New() - g.MulScalar(r, p0, e) + g.MulScalarBig(r, p0, e) // Encode the G1 point into 128 bytes - return g.EncodePoint(r), nil + return encodePointG1(g, r), nil } // bls12381G1MultiExp implements EIP-2537 G1MultiExp precompile. @@ -773,7 +773,7 @@ func (c *bls12381G1MultiExp) Run(input []byte) ([]byte, error) { off := 160 * i t0, t1, t2 := off, off+128, off+160 // Decode G1 point - if points[i], err = g.DecodePoint(input[t0:t1]); err != nil { + if points[i], err = decodePointG1(g, input[t0:t1]); err != nil { return nil, err } // Decode scalar value @@ -782,10 +782,10 @@ func (c *bls12381G1MultiExp) Run(input []byte) ([]byte, error) { // Compute r = e_0 * p_0 + e_1 * p_1 + ... + e_(k-1) * p_(k-1) r := g.New() - g.MultiExp(r, points, scalars) + g.MultiExpBig(r, points, scalars) // Encode the G1 point to 128 bytes - return g.EncodePoint(r), nil + return encodePointG1(g, r), nil } // bls12381G2Add implements EIP-2537 G2Add precompile. @@ -811,11 +811,11 @@ func (c *bls12381G2Add) Run(input []byte) ([]byte, error) { r := g.New() // Decode G2 point p_0 - if p0, err = g.DecodePoint(input[:256]); err != nil { + if p0, err = decodePointG2(g, input[:256]); err != nil { return nil, err } // Decode G2 point p_1 - if p1, err = g.DecodePoint(input[256:]); err != nil { + if p1, err = decodePointG2(g, input[256:]); err != nil { return nil, err } @@ -823,7 +823,7 @@ func (c *bls12381G2Add) Run(input []byte) ([]byte, error) { g.Add(r, p0, p1) // Encode the G2 point into 256 bytes - return g.EncodePoint(r), nil + return encodePointG2(g, r), nil } // bls12381G2Mul implements EIP-2537 G2Mul precompile. @@ -848,7 +848,7 @@ func (c *bls12381G2Mul) Run(input []byte) ([]byte, error) { g := bls12381.NewG2() // Decode G2 point - if p0, err = g.DecodePoint(input[:256]); err != nil { + if p0, err = decodePointG2(g, input[:256]); err != nil { return nil, err } // Decode scalar value @@ -856,10 +856,10 @@ func (c *bls12381G2Mul) Run(input []byte) ([]byte, error) { // Compute r = e * p_0 r := g.New() - g.MulScalar(r, p0, e) + g.MulScalarBig(r, p0, e) // Encode the G2 point into 256 bytes - return g.EncodePoint(r), nil + return encodePointG2(g, r), nil } // bls12381G2MultiExp implements EIP-2537 G2MultiExp precompile. @@ -903,8 +903,8 @@ func (c *bls12381G2MultiExp) Run(input []byte) ([]byte, error) { for i := 0; i < k; i++ { off := 288 * i t0, t1, t2 := off, off+256, off+288 - // Decode G1 point - if points[i], err = g.DecodePoint(input[t0:t1]); err != nil { + // Decode G2 point + if points[i], err = decodePointG2(g, input[t0:t1]); err != nil { return nil, err } // Decode scalar value @@ -913,10 +913,10 @@ func (c *bls12381G2MultiExp) Run(input []byte) ([]byte, error) { // Compute r = e_0 * p_0 + e_1 * p_1 + ... + e_(k-1) * p_(k-1) r := g.New() - g.MultiExp(r, points, scalars) + g.MultiExpBig(r, points, scalars) // Encode the G2 point to 256 bytes. - return g.EncodePoint(r), nil + return encodePointG2(g, r), nil } // bls12381Pairing implements EIP-2537 Pairing precompile. @@ -940,7 +940,7 @@ func (c *bls12381Pairing) Run(input []byte) ([]byte, error) { } // Initialize BLS12-381 pairing engine - e := bls12381.NewPairingEngine() + e := bls12381.NewEngine() g1, g2 := e.G1, e.G2 // Decode pairs @@ -949,12 +949,12 @@ func (c *bls12381Pairing) Run(input []byte) ([]byte, error) { t0, t1, t2 := off, off+128, off+384 // Decode G1 point - p1, err := g1.DecodePoint(input[t0:t1]) + p1, err := decodePointG1(g1, input[t0:t1]) if err != nil { return nil, err } // Decode G2 point - p2, err := g2.DecodePoint(input[t1:t2]) + p2, err := decodePointG2(g2, input[t1:t2]) if err != nil { return nil, err } @@ -981,6 +981,55 @@ func (c *bls12381Pairing) Run(input []byte) ([]byte, error) { return out, nil } +func decodePointG1(g *bls12381.G1, in []byte) (*bls12381.PointG1, error) { + if len(in) != 128 { + return nil, errors.New("invalid g1 point length") + } + pointBytes := make([]byte, 96) + // decode x + xBytes, err := decodeBLS12381FieldElement(in[:64]) + if err != nil { + return nil, err + } + // decode y + yBytes, err := decodeBLS12381FieldElement(in[64:]) + if err != nil { + return nil, err + } + copy(pointBytes[:48], xBytes) + copy(pointBytes[48:], yBytes) + return g.FromBytes(pointBytes) +} + +// decodePointG2 given encoded (x, y) coordinates in 256 bytes returns a valid G2 Point. +func decodePointG2(g *bls12381.G2, in []byte) (*bls12381.PointG2, error) { + if len(in) != 256 { + return nil, errors.New("invalid g2 point length") + } + pointBytes := make([]byte, 192) + x0Bytes, err := decodeBLS12381FieldElement(in[:64]) + if err != nil { + return nil, err + } + x1Bytes, err := decodeBLS12381FieldElement(in[64:128]) + if err != nil { + return nil, err + } + y0Bytes, err := decodeBLS12381FieldElement(in[128:192]) + if err != nil { + return nil, err + } + y1Bytes, err := decodeBLS12381FieldElement(in[192:]) + if err != nil { + return nil, err + } + copy(pointBytes[:48], x1Bytes) + copy(pointBytes[48:96], x0Bytes) + copy(pointBytes[96:144], y1Bytes) + copy(pointBytes[144:192], y0Bytes) + return g.FromBytes(pointBytes) +} + // decodeBLS12381FieldElement decodes BLS12-381 elliptic curve field element. // Removes top 16 bytes of 64 byte input. func decodeBLS12381FieldElement(in []byte) ([]byte, error) { @@ -998,6 +1047,31 @@ func decodeBLS12381FieldElement(in []byte) ([]byte, error) { return out, nil } +// encodePointG1 encodes a point into 128 bytes. +func encodePointG1(g *bls12381.G1, p *bls12381.PointG1) []byte { + outRaw := g.ToBytes(p) + out := make([]byte, 128) + // encode x + copy(out[16:], outRaw[:48]) + // encode y + copy(out[64+16:], outRaw[48:]) + return out +} + +// encodePointG2 encodes a point into 256 bytes. +func encodePointG2(g *bls12381.G2, p *bls12381.PointG2) []byte { + // outRaw is 96 bytes + outRaw := g.ToBytes(p) + out := make([]byte, 256) + // encode x + copy(out[16:16+48], outRaw[48:96]) + copy(out[80:80+48], outRaw[:48]) + // encode y + copy(out[144:144+48], outRaw[144:]) + copy(out[208:208+48], outRaw[96:144]) + return out +} + // bls12381MapG1 implements EIP-2537 MapG1 precompile. type bls12381MapG1 struct{} @@ -1030,7 +1104,7 @@ func (c *bls12381MapG1) Run(input []byte) ([]byte, error) { } // Encode the G1 point to 128 bytes - return g.EncodePoint(r), nil + return encodePointG1(g, r), nil } // bls12381MapG2 implements EIP-2537 MapG2 precompile. @@ -1072,7 +1146,7 @@ func (c *bls12381MapG2) Run(input []byte) ([]byte, error) { } // Encode the G2 point to 256 bytes - return g.EncodePoint(r), nil + return encodePointG2(g, r), nil } // kzgPointEvaluation implements the EIP-4844 point evaluation precompile. diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 6608ff09fc..5c4d2ba61a 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -372,7 +372,7 @@ func BenchmarkPrecompiledBLS12381G1MultiExpWorstCase(b *testing.B) { Name: "WorstCaseG1", NoBenchmark: false, } - benchmarkPrecompiled("0c", testcase, b) + benchmarkPrecompiled("f0c", testcase, b) } // BenchmarkPrecompiledBLS12381G2MultiExpWorstCase benchmarks the worst case we could find that still fits a gaslimit of 10MGas. @@ -393,5 +393,5 @@ func BenchmarkPrecompiledBLS12381G2MultiExpWorstCase(b *testing.B) { Name: "WorstCaseG2", NoBenchmark: false, } - benchmarkPrecompiled("0f", testcase, b) + benchmarkPrecompiled("f0f", testcase, b) } diff --git a/crypto/bls12381/arithmetic_decl.go b/crypto/bls12381/arithmetic_decl.go index f6d232d658..1efd36f73c 100644 --- a/crypto/bls12381/arithmetic_decl.go +++ b/crypto/bls12381/arithmetic_decl.go @@ -1,21 +1,4 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build (amd64 && blsasm) || (amd64 && blsadx) -// +build amd64,blsasm amd64,blsadx +// +build amd64,!generic package bls12381 @@ -24,13 +7,22 @@ import ( ) func init() { - if !enableADX || !cpu.X86.HasADX || !cpu.X86.HasBMI2 { + if !cpu.X86.HasADX || !cpu.X86.HasBMI2 { mul = mulNoADX + wmul = wmulNoADX + fromWide = montRedNoADX + mulFR = mulNoADXFR + wmulFR = wmulNoADXFR + wfp2Mul = wfp2MulGeneric + wfp2Square = wfp2SquareGeneric } } -// Use ADX backend for default var mul func(c, a, b *fe) = mulADX +var wmul func(c *wfe, a, b *fe) = wmulADX +var fromWide func(c *fe, w *wfe) = montRedADX +var wfp2Mul func(c *wfe2, a, b *fe2) = wfp2MulADX +var wfp2Square func(c *wfe2, b *fe2) = wfp2SquareADX func square(c, a *fe) { mul(c, a, a) @@ -65,6 +57,9 @@ func doubleAssign(a *fe) //go:noescape func ldouble(c, a *fe) +//go:noescape +func ldoubleAssign(a *fe) + //go:noescape func sub(c, a, b *fe) @@ -82,3 +77,165 @@ func mulNoADX(c, a, b *fe) //go:noescape func mulADX(c, a, b *fe) + +//go:noescape +func wmulNoADX(c *wfe, a, b *fe) + +//go:noescape +func wmulADX(c *wfe, a, b *fe) + +//go:noescape +func montRedNoADX(a *fe, w *wfe) + +//go:noescape +func montRedADX(a *fe, w *wfe) + +//go:noescape +func lwadd(c, a, b *wfe) + +//go:noescape +func lwaddAssign(a, b *wfe) + +//go:noescape +func wadd(c, a, b *wfe) + +//go:noescape +func lwdouble(c, a *wfe) + +//go:noescape +func wdouble(c, a *wfe) + +//go:noescape +func lwsub(c, a, b *wfe) + +//go:noescape +func lwsubAssign(a, b *wfe) + +//go:noescape +func wsub(c, a, b *wfe) + +//go:noescape +func fp2Add(c, a, b *fe2) + +//go:noescape +func fp2AddAssign(a, b *fe2) + +//go:noescape +func fp2Ladd(c, a, b *fe2) + +//go:noescape +func fp2LaddAssign(a, b *fe2) + +//go:noescape +func fp2DoubleAssign(a *fe2) + +//go:noescape +func fp2Double(c, a *fe2) + +//go:noescape +func fp2Sub(c, a, b *fe2) + +//go:noescape +func fp2SubAssign(a, b *fe2) + +//go:noescape +func mulByNonResidue(c, a *fe2) + +//go:noescape +func mulByNonResidueAssign(a *fe2) + +//go:noescape +func wfp2Add(c, a, b *wfe2) + +//go:noescape +func wfp2AddAssign(a, b *wfe2) + +//go:noescape +func wfp2Ladd(c, a, b *wfe2) + +//go:noescape +func wfp2LaddAssign(a, b *wfe2) + +//go:noescape +func wfp2AddMixed(c, a, b *wfe2) + +//go:noescape +func wfp2AddMixedAssign(a, b *wfe2) + +//go:noescape +func wfp2Sub(c, a, b *wfe2) + +//go:noescape +func wfp2SubAssign(a, b *wfe2) + +//go:noescape +func wfp2SubMixed(c, a, b *wfe2) + +//go:noescape +func wfp2SubMixedAssign(a, b *wfe2) + +//go:noescape +func wfp2Double(c, a *wfe2) + +//go:noescape +func wfp2DoubleAssign(a *wfe2) + +//go:noescape +func wfp2MulByNonResidue(c, a *wfe2) + +//go:noescape +func wfp2MulByNonResidueAssign(a *wfe2) + +//go:noescape +func wfp2SquareADX(c *wfe2, a *fe2) + +//go:noescape +func wfp2MulADX(c *wfe2, a, b *fe2) + +var mulFR func(c, a, b *Fr) = mulADXFR +var wmulFR func(c *wideFr, a, b *Fr) = wmulADXFR + +func squareFR(c, a *Fr) { + mulFR(c, a, a) +} + +func negFR(c, a *Fr) { + if a.IsZero() { + c.Set(a) + } else { + _negFR(c, a) + } +} + +//go:noescape +func addFR(c, a, b *Fr) + +//go:noescape +func laddAssignFR(a, b *Fr) + +//go:noescape +func doubleFR(c, a *Fr) + +//go:noescape +func subFR(c, a, b *Fr) + +//go:noescape +func lsubAssignFR(a, b *Fr) + +//go:noescape +func _negFR(c, a *Fr) + +//go:noescape +func mulNoADXFR(c, a, b *Fr) + +//go:noescape +func mulADXFR(c, a, b *Fr) + +//go:noescape +func wmulADXFR(c *wideFr, a, b *Fr) + +//go:noescape +func wmulNoADXFR(c *wideFr, a, b *Fr) + +//go:noescape +func waddFR(a, b *wideFr) diff --git a/crypto/bls12381/arithmetic_fallback.go b/crypto/bls12381/arithmetic_fallback.go index c09ae0d91c..dd8e3a99d4 100644 --- a/crypto/bls12381/arithmetic_fallback.go +++ b/crypto/bls12381/arithmetic_fallback.go @@ -1,8 +1,6 @@ -// Native go field arithmetic code is generated with 'goff' -// https://github.com/ConsenSys/goff -// Many function signature of field operations are renamed. +// +build !amd64 generic -// Copyright 2020 ConsenSys AG +// Copyright 2020 ConsenSys Software Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,23 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// field modulus q = -// -// 4002409555221667393417789825735904156556882819939007885332058136124031650490837864442687629129015664037894272559787 -// Code generated by goff DO NOT EDIT -// goff version: v0.1.0 - build: 790f1f56eac432441e043abff8819eacddd1d668 -// fe are assumed to be in Montgomery form in all methods - -// /!\ WARNING /!\ -// this code has not been audited and is provided as-is. In particular, -// there is no security guarantees such as constant time implementation -// or side-channel attack resistance -// /!\ WARNING /!\ - -// Package bls (generated by goff) contains field arithmetics operations - -//go:build !amd64 || (!blsasm && !blsadx) -// +build !amd64 !blsasm,!blsadx +// Code generated by goff (v0.3.5) DO NOT EDIT package bls12381 @@ -40,440 +22,6 @@ import ( "math/bits" ) -func add(z, x, y *fe) { - var carry uint64 - - z[0], carry = bits.Add64(x[0], y[0], 0) - z[1], carry = bits.Add64(x[1], y[1], carry) - z[2], carry = bits.Add64(x[2], y[2], carry) - z[3], carry = bits.Add64(x[3], y[3], carry) - z[4], carry = bits.Add64(x[4], y[4], carry) - z[5], _ = bits.Add64(x[5], y[5], carry) - - // if z > q --> z -= q - // note: this is NOT constant time - if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { - var b uint64 - z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) - z[1], b = bits.Sub64(z[1], 2210141511517208575, b) - z[2], b = bits.Sub64(z[2], 7435674573564081700, b) - z[3], b = bits.Sub64(z[3], 7239337960414712511, b) - z[4], b = bits.Sub64(z[4], 5412103778470702295, b) - z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) - } -} - -func addAssign(x, y *fe) { - var carry uint64 - - x[0], carry = bits.Add64(x[0], y[0], 0) - x[1], carry = bits.Add64(x[1], y[1], carry) - x[2], carry = bits.Add64(x[2], y[2], carry) - x[3], carry = bits.Add64(x[3], y[3], carry) - x[4], carry = bits.Add64(x[4], y[4], carry) - x[5], _ = bits.Add64(x[5], y[5], carry) - - // if z > q --> z -= q - // note: this is NOT constant time - if !(x[5] < 1873798617647539866 || (x[5] == 1873798617647539866 && (x[4] < 5412103778470702295 || (x[4] == 5412103778470702295 && (x[3] < 7239337960414712511 || (x[3] == 7239337960414712511 && (x[2] < 7435674573564081700 || (x[2] == 7435674573564081700 && (x[1] < 2210141511517208575 || (x[1] == 2210141511517208575 && (x[0] < 13402431016077863595))))))))))) { - var b uint64 - x[0], b = bits.Sub64(x[0], 13402431016077863595, 0) - x[1], b = bits.Sub64(x[1], 2210141511517208575, b) - x[2], b = bits.Sub64(x[2], 7435674573564081700, b) - x[3], b = bits.Sub64(x[3], 7239337960414712511, b) - x[4], b = bits.Sub64(x[4], 5412103778470702295, b) - x[5], _ = bits.Sub64(x[5], 1873798617647539866, b) - } -} - -func ladd(z, x, y *fe) { - var carry uint64 - z[0], carry = bits.Add64(x[0], y[0], 0) - z[1], carry = bits.Add64(x[1], y[1], carry) - z[2], carry = bits.Add64(x[2], y[2], carry) - z[3], carry = bits.Add64(x[3], y[3], carry) - z[4], carry = bits.Add64(x[4], y[4], carry) - z[5], _ = bits.Add64(x[5], y[5], carry) -} - -func laddAssign(x, y *fe) { - var carry uint64 - x[0], carry = bits.Add64(x[0], y[0], 0) - x[1], carry = bits.Add64(x[1], y[1], carry) - x[2], carry = bits.Add64(x[2], y[2], carry) - x[3], carry = bits.Add64(x[3], y[3], carry) - x[4], carry = bits.Add64(x[4], y[4], carry) - x[5], _ = bits.Add64(x[5], y[5], carry) -} - -func double(z, x *fe) { - var carry uint64 - - z[0], carry = bits.Add64(x[0], x[0], 0) - z[1], carry = bits.Add64(x[1], x[1], carry) - z[2], carry = bits.Add64(x[2], x[2], carry) - z[3], carry = bits.Add64(x[3], x[3], carry) - z[4], carry = bits.Add64(x[4], x[4], carry) - z[5], _ = bits.Add64(x[5], x[5], carry) - - // if z > q --> z -= q - // note: this is NOT constant time - if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { - var b uint64 - z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) - z[1], b = bits.Sub64(z[1], 2210141511517208575, b) - z[2], b = bits.Sub64(z[2], 7435674573564081700, b) - z[3], b = bits.Sub64(z[3], 7239337960414712511, b) - z[4], b = bits.Sub64(z[4], 5412103778470702295, b) - z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) - } -} - -func doubleAssign(z *fe) { - var carry uint64 - - z[0], carry = bits.Add64(z[0], z[0], 0) - z[1], carry = bits.Add64(z[1], z[1], carry) - z[2], carry = bits.Add64(z[2], z[2], carry) - z[3], carry = bits.Add64(z[3], z[3], carry) - z[4], carry = bits.Add64(z[4], z[4], carry) - z[5], _ = bits.Add64(z[5], z[5], carry) - - // if z > q --> z -= q - // note: this is NOT constant time - if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { - var b uint64 - z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) - z[1], b = bits.Sub64(z[1], 2210141511517208575, b) - z[2], b = bits.Sub64(z[2], 7435674573564081700, b) - z[3], b = bits.Sub64(z[3], 7239337960414712511, b) - z[4], b = bits.Sub64(z[4], 5412103778470702295, b) - z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) - } -} - -func ldouble(z, x *fe) { - var carry uint64 - - z[0], carry = bits.Add64(x[0], x[0], 0) - z[1], carry = bits.Add64(x[1], x[1], carry) - z[2], carry = bits.Add64(x[2], x[2], carry) - z[3], carry = bits.Add64(x[3], x[3], carry) - z[4], carry = bits.Add64(x[4], x[4], carry) - z[5], _ = bits.Add64(x[5], x[5], carry) -} - -func sub(z, x, y *fe) { - var b uint64 - z[0], b = bits.Sub64(x[0], y[0], 0) - z[1], b = bits.Sub64(x[1], y[1], b) - z[2], b = bits.Sub64(x[2], y[2], b) - z[3], b = bits.Sub64(x[3], y[3], b) - z[4], b = bits.Sub64(x[4], y[4], b) - z[5], b = bits.Sub64(x[5], y[5], b) - if b != 0 { - var c uint64 - z[0], c = bits.Add64(z[0], 13402431016077863595, 0) - z[1], c = bits.Add64(z[1], 2210141511517208575, c) - z[2], c = bits.Add64(z[2], 7435674573564081700, c) - z[3], c = bits.Add64(z[3], 7239337960414712511, c) - z[4], c = bits.Add64(z[4], 5412103778470702295, c) - z[5], _ = bits.Add64(z[5], 1873798617647539866, c) - } -} - -func subAssign(z, x *fe) { - var b uint64 - z[0], b = bits.Sub64(z[0], x[0], 0) - z[1], b = bits.Sub64(z[1], x[1], b) - z[2], b = bits.Sub64(z[2], x[2], b) - z[3], b = bits.Sub64(z[3], x[3], b) - z[4], b = bits.Sub64(z[4], x[4], b) - z[5], b = bits.Sub64(z[5], x[5], b) - if b != 0 { - var c uint64 - z[0], c = bits.Add64(z[0], 13402431016077863595, 0) - z[1], c = bits.Add64(z[1], 2210141511517208575, c) - z[2], c = bits.Add64(z[2], 7435674573564081700, c) - z[3], c = bits.Add64(z[3], 7239337960414712511, c) - z[4], c = bits.Add64(z[4], 5412103778470702295, c) - z[5], _ = bits.Add64(z[5], 1873798617647539866, c) - } -} - -func lsubAssign(z, x *fe) { - var b uint64 - z[0], b = bits.Sub64(z[0], x[0], 0) - z[1], b = bits.Sub64(z[1], x[1], b) - z[2], b = bits.Sub64(z[2], x[2], b) - z[3], b = bits.Sub64(z[3], x[3], b) - z[4], b = bits.Sub64(z[4], x[4], b) - z[5], _ = bits.Sub64(z[5], x[5], b) -} - -func neg(z *fe, x *fe) { - if x.isZero() { - z.zero() - return - } - var borrow uint64 - z[0], borrow = bits.Sub64(13402431016077863595, x[0], 0) - z[1], borrow = bits.Sub64(2210141511517208575, x[1], borrow) - z[2], borrow = bits.Sub64(7435674573564081700, x[2], borrow) - z[3], borrow = bits.Sub64(7239337960414712511, x[3], borrow) - z[4], borrow = bits.Sub64(5412103778470702295, x[4], borrow) - z[5], _ = bits.Sub64(1873798617647539866, x[5], borrow) -} - -func mul(z, x, y *fe) { - var t [6]uint64 - var c [3]uint64 - { - // round 0 - v := x[0] - c[1], c[0] = bits.Mul64(v, y[0]) - m := c[0] * 9940570264628428797 - c[2] = madd0(m, 13402431016077863595, c[0]) - c[1], c[0] = madd1(v, y[1], c[1]) - c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) - c[1], c[0] = madd1(v, y[2], c[1]) - c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) - c[1], c[0] = madd1(v, y[3], c[1]) - c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) - c[1], c[0] = madd1(v, y[4], c[1]) - c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) - c[1], c[0] = madd1(v, y[5], c[1]) - t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) - } - { - // round 1 - v := x[1] - c[1], c[0] = madd1(v, y[0], t[0]) - m := c[0] * 9940570264628428797 - c[2] = madd0(m, 13402431016077863595, c[0]) - c[1], c[0] = madd2(v, y[1], c[1], t[1]) - c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) - c[1], c[0] = madd2(v, y[2], c[1], t[2]) - c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) - c[1], c[0] = madd2(v, y[3], c[1], t[3]) - c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) - c[1], c[0] = madd2(v, y[4], c[1], t[4]) - c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) - c[1], c[0] = madd2(v, y[5], c[1], t[5]) - t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) - } - { - // round 2 - v := x[2] - c[1], c[0] = madd1(v, y[0], t[0]) - m := c[0] * 9940570264628428797 - c[2] = madd0(m, 13402431016077863595, c[0]) - c[1], c[0] = madd2(v, y[1], c[1], t[1]) - c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) - c[1], c[0] = madd2(v, y[2], c[1], t[2]) - c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) - c[1], c[0] = madd2(v, y[3], c[1], t[3]) - c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) - c[1], c[0] = madd2(v, y[4], c[1], t[4]) - c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) - c[1], c[0] = madd2(v, y[5], c[1], t[5]) - t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) - } - { - // round 3 - v := x[3] - c[1], c[0] = madd1(v, y[0], t[0]) - m := c[0] * 9940570264628428797 - c[2] = madd0(m, 13402431016077863595, c[0]) - c[1], c[0] = madd2(v, y[1], c[1], t[1]) - c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) - c[1], c[0] = madd2(v, y[2], c[1], t[2]) - c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) - c[1], c[0] = madd2(v, y[3], c[1], t[3]) - c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) - c[1], c[0] = madd2(v, y[4], c[1], t[4]) - c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) - c[1], c[0] = madd2(v, y[5], c[1], t[5]) - t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) - } - { - // round 4 - v := x[4] - c[1], c[0] = madd1(v, y[0], t[0]) - m := c[0] * 9940570264628428797 - c[2] = madd0(m, 13402431016077863595, c[0]) - c[1], c[0] = madd2(v, y[1], c[1], t[1]) - c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) - c[1], c[0] = madd2(v, y[2], c[1], t[2]) - c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) - c[1], c[0] = madd2(v, y[3], c[1], t[3]) - c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) - c[1], c[0] = madd2(v, y[4], c[1], t[4]) - c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) - c[1], c[0] = madd2(v, y[5], c[1], t[5]) - t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) - } - { - // round 5 - v := x[5] - c[1], c[0] = madd1(v, y[0], t[0]) - m := c[0] * 9940570264628428797 - c[2] = madd0(m, 13402431016077863595, c[0]) - c[1], c[0] = madd2(v, y[1], c[1], t[1]) - c[2], z[0] = madd2(m, 2210141511517208575, c[2], c[0]) - c[1], c[0] = madd2(v, y[2], c[1], t[2]) - c[2], z[1] = madd2(m, 7435674573564081700, c[2], c[0]) - c[1], c[0] = madd2(v, y[3], c[1], t[3]) - c[2], z[2] = madd2(m, 7239337960414712511, c[2], c[0]) - c[1], c[0] = madd2(v, y[4], c[1], t[4]) - c[2], z[3] = madd2(m, 5412103778470702295, c[2], c[0]) - c[1], c[0] = madd2(v, y[5], c[1], t[5]) - z[5], z[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) - } - - // if z > q --> z -= q - // note: this is NOT constant time - if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { - var b uint64 - z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) - z[1], b = bits.Sub64(z[1], 2210141511517208575, b) - z[2], b = bits.Sub64(z[2], 7435674573564081700, b) - z[3], b = bits.Sub64(z[3], 7239337960414712511, b) - z[4], b = bits.Sub64(z[4], 5412103778470702295, b) - z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) - } -} - -func square(z, x *fe) { - - var p [6]uint64 - - var u, v uint64 - { - // round 0 - u, p[0] = bits.Mul64(x[0], x[0]) - m := p[0] * 9940570264628428797 - C := madd0(m, 13402431016077863595, p[0]) - var t uint64 - t, u, v = madd1sb(x[0], x[1], u) - C, p[0] = madd2(m, 2210141511517208575, v, C) - t, u, v = madd1s(x[0], x[2], t, u) - C, p[1] = madd2(m, 7435674573564081700, v, C) - t, u, v = madd1s(x[0], x[3], t, u) - C, p[2] = madd2(m, 7239337960414712511, v, C) - t, u, v = madd1s(x[0], x[4], t, u) - C, p[3] = madd2(m, 5412103778470702295, v, C) - _, u, v = madd1s(x[0], x[5], t, u) - p[5], p[4] = madd3(m, 1873798617647539866, v, C, u) - } - { - // round 1 - m := p[0] * 9940570264628428797 - C := madd0(m, 13402431016077863595, p[0]) - u, v = madd1(x[1], x[1], p[1]) - C, p[0] = madd2(m, 2210141511517208575, v, C) - var t uint64 - t, u, v = madd2sb(x[1], x[2], p[2], u) - C, p[1] = madd2(m, 7435674573564081700, v, C) - t, u, v = madd2s(x[1], x[3], p[3], t, u) - C, p[2] = madd2(m, 7239337960414712511, v, C) - t, u, v = madd2s(x[1], x[4], p[4], t, u) - C, p[3] = madd2(m, 5412103778470702295, v, C) - _, u, v = madd2s(x[1], x[5], p[5], t, u) - p[5], p[4] = madd3(m, 1873798617647539866, v, C, u) - } - { - // round 2 - m := p[0] * 9940570264628428797 - C := madd0(m, 13402431016077863595, p[0]) - C, p[0] = madd2(m, 2210141511517208575, p[1], C) - u, v = madd1(x[2], x[2], p[2]) - C, p[1] = madd2(m, 7435674573564081700, v, C) - var t uint64 - t, u, v = madd2sb(x[2], x[3], p[3], u) - C, p[2] = madd2(m, 7239337960414712511, v, C) - t, u, v = madd2s(x[2], x[4], p[4], t, u) - C, p[3] = madd2(m, 5412103778470702295, v, C) - _, u, v = madd2s(x[2], x[5], p[5], t, u) - p[5], p[4] = madd3(m, 1873798617647539866, v, C, u) - } - { - // round 3 - m := p[0] * 9940570264628428797 - C := madd0(m, 13402431016077863595, p[0]) - C, p[0] = madd2(m, 2210141511517208575, p[1], C) - C, p[1] = madd2(m, 7435674573564081700, p[2], C) - u, v = madd1(x[3], x[3], p[3]) - C, p[2] = madd2(m, 7239337960414712511, v, C) - var t uint64 - t, u, v = madd2sb(x[3], x[4], p[4], u) - C, p[3] = madd2(m, 5412103778470702295, v, C) - _, u, v = madd2s(x[3], x[5], p[5], t, u) - p[5], p[4] = madd3(m, 1873798617647539866, v, C, u) - } - { - // round 4 - m := p[0] * 9940570264628428797 - C := madd0(m, 13402431016077863595, p[0]) - C, p[0] = madd2(m, 2210141511517208575, p[1], C) - C, p[1] = madd2(m, 7435674573564081700, p[2], C) - C, p[2] = madd2(m, 7239337960414712511, p[3], C) - u, v = madd1(x[4], x[4], p[4]) - C, p[3] = madd2(m, 5412103778470702295, v, C) - _, u, v = madd2sb(x[4], x[5], p[5], u) - p[5], p[4] = madd3(m, 1873798617647539866, v, C, u) - } - { - // round 5 - m := p[0] * 9940570264628428797 - C := madd0(m, 13402431016077863595, p[0]) - C, z[0] = madd2(m, 2210141511517208575, p[1], C) - C, z[1] = madd2(m, 7435674573564081700, p[2], C) - C, z[2] = madd2(m, 7239337960414712511, p[3], C) - C, z[3] = madd2(m, 5412103778470702295, p[4], C) - u, v = madd1(x[5], x[5], p[5]) - z[5], z[4] = madd3(m, 1873798617647539866, v, C, u) - } - - // if z > q --> z -= q - // note: this is NOT constant time - if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { - var b uint64 - z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) - z[1], b = bits.Sub64(z[1], 2210141511517208575, b) - z[2], b = bits.Sub64(z[2], 7435674573564081700, b) - z[3], b = bits.Sub64(z[3], 7239337960414712511, b) - z[4], b = bits.Sub64(z[4], 5412103778470702295, b) - z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) - } -} - -// arith.go -// Copyright 2020 ConsenSys AG -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Code generated by goff DO NOT EDIT - -func madd(a, b, t, u, v uint64) (uint64, uint64, uint64) { - var carry uint64 - hi, lo := bits.Mul64(a, b) - v, carry = bits.Add64(lo, v, 0) - u, carry = bits.Add64(hi, u, carry) - t, _ = bits.Add64(t, 0, carry) - return t, u, v -} - // madd0 hi = a*b + c (discards lo bits) func madd0(a, b, c uint64) (hi uint64) { var carry, lo uint64 @@ -503,59 +51,6 @@ func madd2(a, b, c, d uint64) (hi uint64, lo uint64) { return } -// madd2s superhi, hi, lo = 2*a*b + c + d + e -func madd2s(a, b, c, d, e uint64) (superhi, hi, lo uint64) { - var carry, sum uint64 - - hi, lo = bits.Mul64(a, b) - lo, carry = bits.Add64(lo, lo, 0) - hi, superhi = bits.Add64(hi, hi, carry) - - sum, carry = bits.Add64(c, e, 0) - hi, _ = bits.Add64(hi, 0, carry) - lo, carry = bits.Add64(lo, sum, 0) - hi, _ = bits.Add64(hi, 0, carry) - hi, _ = bits.Add64(hi, 0, d) - return -} - -func madd1s(a, b, d, e uint64) (superhi, hi, lo uint64) { - var carry uint64 - - hi, lo = bits.Mul64(a, b) - lo, carry = bits.Add64(lo, lo, 0) - hi, superhi = bits.Add64(hi, hi, carry) - lo, carry = bits.Add64(lo, e, 0) - hi, _ = bits.Add64(hi, 0, carry) - hi, _ = bits.Add64(hi, 0, d) - return -} - -func madd2sb(a, b, c, e uint64) (superhi, hi, lo uint64) { - var carry, sum uint64 - - hi, lo = bits.Mul64(a, b) - lo, carry = bits.Add64(lo, lo, 0) - hi, superhi = bits.Add64(hi, hi, carry) - - sum, carry = bits.Add64(c, e, 0) - hi, _ = bits.Add64(hi, 0, carry) - lo, carry = bits.Add64(lo, sum, 0) - hi, _ = bits.Add64(hi, 0, carry) - return -} - -func madd1sb(a, b, e uint64) (superhi, hi, lo uint64) { - var carry uint64 - - hi, lo = bits.Mul64(a, b) - lo, carry = bits.Add64(lo, lo, 0) - hi, superhi = bits.Add64(hi, hi, carry) - lo, carry = bits.Add64(lo, e, 0) - hi, _ = bits.Add64(hi, 0, carry) - return -} - func madd3(a, b, c, d, e uint64) (hi uint64, lo uint64) { var carry uint64 hi, lo = bits.Mul64(a, b) diff --git a/crypto/bls12381/bls12_381.go b/crypto/bls12381/bls12_381.go index 1c1c97765f..0aa080d09d 100644 --- a/crypto/bls12381/bls12_381.go +++ b/crypto/bls12381/bls12_381.go @@ -1,79 +1,89 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 -/* - Field Constants -*/ +const fpNumberOfLimbs = 6 +const fpByteSize = 48 +const fpBitSize = 381 +const sixWordBitSize = 384 -// Base field modulus +// Base Field // p = 0x1a0111ea397fe69a4b1ba7b6434bacd764774b84f38512bf6730d2a0f6b0f6241eabfffeb153ffffb9feffffffffaaab - -// Size of six words // r = 2 ^ 384 // modulus = p var modulus = fe{0xb9feffffffffaaab, 0x1eabfffeb153ffff, 0x6730d2a0f6b0f624, 0x64774b84f38512bf, 0x4b1ba7b6434bacd7, 0x1a0111ea397fe69a} -var ( - // -p^(-1) mod 2^64 - inp uint64 = 0x89f3fffcfffcfffd - // This value is used in assembly code - _ = inp -) +// -p^(-1) mod 2^64 +var inp uint64 = 0x89f3fffcfffcfffd -// r mod p +// r1 = r mod p var r1 = &fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493} -// r^2 mod p +// one = mod p +var one = r1 + +// zero = 0 +var zero = &fe{} + +// r2 = r^2 mod p var r2 = &fe{ 0xf4df1f341c341746, 0x0a76e6a609d104f1, 0x8de5476c4c95b6d5, 0x67eb88a9939d83c0, 0x9a793e85b519952d, 0x11988fe592cae3aa, } -// -1 + 0 * u +// negativeOne = -r mod p +var negativeOne = &fe{ + 0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206, +} + +// negativeOne2 = -1 + 0 * u var negativeOne2 = &fe2{ fe{0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206}, fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, } -// 2 ^ (-1) +// twoInv = 2^(-1) var twoInv = &fe{0x1804000000015554, 0x855000053ab00001, 0x633cb57c253c276f, 0x6e22d1ec31ebb502, 0xd3916126f2d14ca2, 0x17fbb8571a006596} -// (p - 3) / 4 +// pMinus3Over4 = (p - 3) / 4 var pMinus3Over4 = bigFromHex("0x680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffffeaaa") -// (p + 1) / 4 +// pPlus1Over4 = (p + 1) / 4 var pPlus1Over4 = bigFromHex("0x680447a8e5ff9a692c6e9ed90d2eb35d91dd2e13ce144afd9cc34a83dac3d8907aaffffac54ffffee7fbfffffffeaab") -// (p - 1) / 2 +// pMinus1Over2 = (p - 1) / 2 var pMinus1Over2 = bigFromHex("0xd0088f51cbff34d258dd3db21a5d66bb23ba5c279c2895fb39869507b587b120f55ffff58a9ffffdcff7fffffffd555") -// -1 +// nonResidue1 = -1 var nonResidue1 = &fe{0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206} -// (1 + 1 * u) +// nonResidue2 = (1 + 1 * u) var nonResidue2 = &fe2{ fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, } -/* - Curve Constants -*/ +// Scalar Field +// q = 0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001 +// Size of six words +// qr = 2 ^ 256 + +var qBig = bigFromHex("0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001") +var q = Fr{0xffffffff00000001, 0x53bda402fffe5bfe, 0x3339d80809a1d805, 0x73eda753299d7d48} + +// var qmodulus = Fr{0xffffffff00000001, 0x53bda402fffe5bfe, 0x3339d80809a1d805, 0x73eda753299d7d48} + +// -q^(-1) mod 2^64 +var qinp uint64 = 0xfffffffeffffffff + +// supress warning: qinp is used in assembly code +var _ = qinp + +// qr1 = qr mod q +var qr1 = &Fr{0x00000001fffffffe, 0x5884b7fa00034802, 0x998c4fefecbc4ff5, 0x1824b159acc5056f} + +// qr2 = qr^2 mod q +var qr2 = &Fr{0xc999e990f3f29c6d, 0x2b6cedcb87925c23, 0x05d314967254398f, 0x0748d9d99f59ff11} + +// Curve Constants // b coefficient for G1 var b = &fe{0xaa270000000cfff3, 0x53cc0032fc34000a, 0x478fe97a6b0a807f, 0xb1d37ebee6ba24d7, 0x8ec9733bbf78ab2f, 0x09d645513d83de7e} @@ -84,21 +94,28 @@ var b2 = &fe2{ fe{0xaa270000000cfff3, 0x53cc0032fc34000a, 0x478fe97a6b0a807f, 0xb1d37ebee6ba24d7, 0x8ec9733bbf78ab2f, 0x09d645513d83de7e}, } -// Curve order -var q = bigFromHex("0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001") +// G1 cofactor +var cofactorG1 = bigFromHex("0x396c8c005555e1568c00aaab0000aaab") -// Efficient cofactor of G1 +// G2 cofactor +var cofactorG2 = bigFromHex("5d543a95414e7f1091d50792876a202cd91de4547085abaa68a205b2e5a7ddfa628f1cb4d9e82ef21537e293a6691ae1616ec6e786f0c70cf1c38e31c7238e5") + +// Efficient G1 cofactor var cofactorEFFG1 = bigFromHex("0xd201000000010001") -// Efficient cofactor of G2 +// Efficient G2 cofactor var cofactorEFFG2 = bigFromHex("0x0bc69f08f2ee75b3584c6a0ea91b352888e2a8e9145ad7689986ff031508ffe1329c2f178731db956d82bf015d1212b02ec0ec69d7477c1ae954cbc06689f6a359894c0adebbf6b4e8020005aaa95551") +// G1 generator var g1One = PointG1{ fe{0x5cb38790fd530c16, 0x7817fc679976fff5, 0x154f95c7143ba1c1, 0xf0ae6acdf3d0e747, 0xedce6ecc21dbf440, 0x120177419e0bfb75}, fe{0xbaac93d50ce72271, 0x8c22631a7918fd8e, 0xdd595f13570725ce, 0x51ac582950405194, 0x0e1c8c3fad0059c0, 0x0bbc3efc5008a26a}, fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, } +var G1One = g1One + +// G2 generator var g2One = PointG2{ fe2{ fe{0xf5f28fa202940a10, 0xb3f5fb2687b4961a, 0xa1a893b53e2ae580, 0x9894999d1a3caee9, 0x6f67b7631863366b, 0x058191924350bcd7}, @@ -114,117 +131,179 @@ var g2One = PointG2{ }, } -/* - Frobenious Coeffs -*/ +var G2One = g2One +// Psi values for faster cofactor clearing + +// psix = 1 / (nr ^ (p - 1)/3) +var psix = fe2{ + fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + fe{0x890dc9e4867545c3, 0x2af322533285a5d5, 0x50880866309b7e2c, 0xa20d1b8c7e881024, 0x14e4f04fe2db9068, 0x14e56d3f1564853a}, +} + +// psiy = 1 / (nr ^ (p - 1)/2) +var psiy = fe2{ + fe{0x3e2f585da55c9ad1, 0x4294213d86c18183, 0x382844c88b623732, 0x92ad2afd19103e18, 0x1d794e4fac7cf0b9, 0x0bd592fc7d825ec8}, + fe{0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, +} + +// Frobenius Coeffs + +// z = -1 +var frobeniusCoeffs2 = [2]fe{ + // z ^ (( p ^ 0 - 1) / 2) + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + // z ^ (( p ^ 1 - 1) / 2) + {0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206}, +} + +// z = u + 1 var frobeniusCoeffs61 = [6]fe2{ + // z ^ (( p ^ 0 - 1) / 3) { - fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ (( p ^ 1 - 1) / 3) { - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, - fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, }, + // z ^ (( p ^ 2 - 1) / 3) { - fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ (( p ^ 3 - 1) / 3) { - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, - fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, }, + // z ^ (( p ^ 4 - 1) / 3) { - fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ (( p ^ 5 - 1) / 3) { - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, - fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, }, } +// z = u + 1 var frobeniusCoeffs62 = [6]fe2{ + // z ^ (( 2 * p ^ 0 - 2) / 3) { - fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ (( 2 * p ^ 1 - 2) / 3) { - fe{0x890dc9e4867545c3, 0x2af322533285a5d5, 0x50880866309b7e2c, 0xa20d1b8c7e881024, 0x14e4f04fe2db9068, 0x14e56d3f1564853a}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x890dc9e4867545c3, 0x2af322533285a5d5, 0x50880866309b7e2c, 0xa20d1b8c7e881024, 0x14e4f04fe2db9068, 0x14e56d3f1564853a}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ (( 2 * p ^ 2 - 2) / 3) { - fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ (( 2 * p ^ 3 - 2) / 3) { - fe{0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ (( 2 * p ^ 4 - 2) / 3) { - fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ (( 2 * p ^ 5 - 2) / 3) { - fe{0xecfb361b798dba3a, 0xc100ddb891865a2c, 0x0ec08ff1232bda8e, 0xd5c13cc6f1ca4721, 0x47222a47bf7b5c04, 0x0110f184e51c5f59}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0xecfb361b798dba3a, 0xc100ddb891865a2c, 0x0ec08ff1232bda8e, 0xd5c13cc6f1ca4721, 0x47222a47bf7b5c04, 0x0110f184e51c5f59}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, } var frobeniusCoeffs12 = [12]fe2{ + // z = u + 1 + // z ^ ((p ^ 0 - 1) / 6) { - fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ ((p ^ 1 - 1) / 6) { - fe{0x07089552b319d465, 0xc6695f92b50a8313, 0x97e83cccd117228f, 0xa35baecab2dc29ee, 0x1ce393ea5daace4d, 0x08f2220fb0fb66eb}, - fe{0xb2f66aad4ce5d646, 0x5842a06bfc497cec, 0xcf4895d42599d394, 0xc11b9cba40a8e8d0, 0x2e3813cbe5a0de89, 0x110eefda88847faf}, + {0x07089552b319d465, 0xc6695f92b50a8313, 0x97e83cccd117228f, 0xa35baecab2dc29ee, 0x1ce393ea5daace4d, 0x08f2220fb0fb66eb}, + {0xb2f66aad4ce5d646, 0x5842a06bfc497cec, 0xcf4895d42599d394, 0xc11b9cba40a8e8d0, 0x2e3813cbe5a0de89, 0x110eefda88847faf}, }, + // z ^ ((p ^ 2 - 1) / 6) { - fe{0xecfb361b798dba3a, 0xc100ddb891865a2c, 0x0ec08ff1232bda8e, 0xd5c13cc6f1ca4721, 0x47222a47bf7b5c04, 0x0110f184e51c5f59}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0xecfb361b798dba3a, 0xc100ddb891865a2c, 0x0ec08ff1232bda8e, 0xd5c13cc6f1ca4721, 0x47222a47bf7b5c04, 0x0110f184e51c5f59}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ ((p ^ 3 - 1) / 6) { - fe{0x3e2f585da55c9ad1, 0x4294213d86c18183, 0x382844c88b623732, 0x92ad2afd19103e18, 0x1d794e4fac7cf0b9, 0x0bd592fc7d825ec8}, - fe{0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, + {0x3e2f585da55c9ad1, 0x4294213d86c18183, 0x382844c88b623732, 0x92ad2afd19103e18, 0x1d794e4fac7cf0b9, 0x0bd592fc7d825ec8}, + {0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, }, + // z ^ ((p ^ 4 - 1) / 6) { - fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ ((p ^ 5 - 1) / 6) { - fe{0x3726c30af242c66c, 0x7c2ac1aad1b6fe70, 0xa04007fbba4b14a2, 0xef517c3266341429, 0x0095ba654ed2226b, 0x02e370eccc86f7dd}, - fe{0x82d83cf50dbce43f, 0xa2813e53df9d018f, 0xc6f0caa53c65e181, 0x7525cf528d50fe95, 0x4a85ed50f4798a6b, 0x171da0fd6cf8eebd}, + {0x3726c30af242c66c, 0x7c2ac1aad1b6fe70, 0xa04007fbba4b14a2, 0xef517c3266341429, 0x0095ba654ed2226b, 0x02e370eccc86f7dd}, + {0x82d83cf50dbce43f, 0xa2813e53df9d018f, 0xc6f0caa53c65e181, 0x7525cf528d50fe95, 0x4a85ed50f4798a6b, 0x171da0fd6cf8eebd}, }, + // z ^ ((p ^ 6 - 1) / 6) { - fe{0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x43f5fffffffcaaae, 0x32b7fff2ed47fffd, 0x07e83a49a2e99d69, 0xeca8f3318332bb7a, 0xef148d1ea0f4c069, 0x040ab3263eff0206}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ ((p ^ 7 - 1) / 6) { - fe{0xb2f66aad4ce5d646, 0x5842a06bfc497cec, 0xcf4895d42599d394, 0xc11b9cba40a8e8d0, 0x2e3813cbe5a0de89, 0x110eefda88847faf}, - fe{0x07089552b319d465, 0xc6695f92b50a8313, 0x97e83cccd117228f, 0xa35baecab2dc29ee, 0x1ce393ea5daace4d, 0x08f2220fb0fb66eb}, + {0xb2f66aad4ce5d646, 0x5842a06bfc497cec, 0xcf4895d42599d394, 0xc11b9cba40a8e8d0, 0x2e3813cbe5a0de89, 0x110eefda88847faf}, + {0x07089552b319d465, 0xc6695f92b50a8313, 0x97e83cccd117228f, 0xa35baecab2dc29ee, 0x1ce393ea5daace4d, 0x08f2220fb0fb66eb}, }, + // z ^ ((p ^ 8 - 1) / 6) { - fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ ((p ^ 9 - 1) / 6) { - fe{0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, - fe{0x3e2f585da55c9ad1, 0x4294213d86c18183, 0x382844c88b623732, 0x92ad2afd19103e18, 0x1d794e4fac7cf0b9, 0x0bd592fc7d825ec8}, + {0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, + {0x3e2f585da55c9ad1, 0x4294213d86c18183, 0x382844c88b623732, 0x92ad2afd19103e18, 0x1d794e4fac7cf0b9, 0x0bd592fc7d825ec8}, }, + // z ^ ((p ^ 10 - 1) / 6) { - fe{0x890dc9e4867545c3, 0x2af322533285a5d5, 0x50880866309b7e2c, 0xa20d1b8c7e881024, 0x14e4f04fe2db9068, 0x14e56d3f1564853a}, - fe{0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, + {0x890dc9e4867545c3, 0x2af322533285a5d5, 0x50880866309b7e2c, 0xa20d1b8c7e881024, 0x14e4f04fe2db9068, 0x14e56d3f1564853a}, + {0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000, 0x0000000000000000}, }, + // z ^ ((p ^ 11 - 1) / 6) { - fe{0x82d83cf50dbce43f, 0xa2813e53df9d018f, 0xc6f0caa53c65e181, 0x7525cf528d50fe95, 0x4a85ed50f4798a6b, 0x171da0fd6cf8eebd}, - fe{0x3726c30af242c66c, 0x7c2ac1aad1b6fe70, 0xa04007fbba4b14a2, 0xef517c3266341429, 0x0095ba654ed2226b, 0x02e370eccc86f7dd}, + {0x82d83cf50dbce43f, 0xa2813e53df9d018f, 0xc6f0caa53c65e181, 0x7525cf528d50fe95, 0x4a85ed50f4798a6b, 0x171da0fd6cf8eebd}, + {0x3726c30af242c66c, 0x7c2ac1aad1b6fe70, 0xa04007fbba4b14a2, 0xef517c3266341429, 0x0095ba654ed2226b, 0x02e370eccc86f7dd}, }, } -/* - x -*/ +// x -var x = bigFromHex("0xd201000000010000") +// var x = bigFromHex("0xd201000000010000") +var x uint64 = 0xd201000000010000 + +// square root + +var sqrtMinus1 = &fe2{*new(fe).zero(), *new(fe).one()} + +var sqrtSqrtMinus1 = &fe2{ + fe{0x3e2f585da55c9ad1, 0x4294213d86c18183, 0x382844c88b623732, 0x92ad2afd19103e18, 0x1d794e4fac7cf0b9, 0x0bd592fc7d825ec8}, + fe{0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, +} + +var sqrtMinusSqrtMinus1 = &fe2{ + fe{0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, + fe{0x7bcfa7a25aa30fda, 0xdc17dec12a927e7c, 0x2f088dd86b4ebef1, 0xd1ca2087da74d4a7, 0x2da2596696cebc1d, 0x0e2b7eedbbfd87d2}, +} diff --git a/crypto/bls12381/bls12_381_test.go b/crypto/bls12381/bls12_381_test.go index 6bf5834105..ae9b8d7fc0 100644 --- a/crypto/bls12381/bls12_381_test.go +++ b/crypto/bls12381/bls12_381_test.go @@ -2,12 +2,66 @@ package bls12381 import ( "crypto/rand" + "encoding/hex" + "errors" + "flag" "math/big" + "os" + "testing" ) -var fuz = 10 +var fuz int + +func TestMain(m *testing.M) { + _fuz := flag.Int("fuzz", 10, "# of iterations") + flag.Parse() + fuz = *_fuz + os.Exit(m.Run()) +} func randScalar(max *big.Int) *big.Int { - a, _ := rand.Int(rand.Reader, max) + a, err := rand.Int(rand.Reader, max) + if err != nil { + panic(errors.New("")) + } return a } + +func fromHex(size int, hexStrs ...string) []byte { + var out []byte + if size > 0 { + out = make([]byte, size*len(hexStrs)) + } + for i := 0; i < len(hexStrs); i++ { + hexStr := hexStrs[i] + if hexStr[:2] == "0x" { + hexStr = hexStr[2:] + } + if len(hexStr)%2 == 1 { + hexStr = "0" + hexStr + } + bytes, err := hex.DecodeString(hexStr) + if err != nil { + return nil + } + if size <= 0 { + out = append(out, bytes...) + } else { + if len(bytes) > size { + return nil + } + offset := i*size + (size - len(bytes)) + copy(out[offset:], bytes) + } + } + return out +} + +func padBytes(in []byte, size int) []byte { + out := make([]byte, size) + if len(in) > size { + panic("bad input for padding") + } + copy(out[size-len(in):], in) + return out +} diff --git a/crypto/bls12381/field_element.go b/crypto/bls12381/field_element.go index 9fdddc6184..4d3e92217c 100644 --- a/crypto/bls12381/field_element.go +++ b/crypto/bls12381/field_element.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( @@ -25,31 +9,34 @@ import ( ) // fe is base field element representation -type fe [6]uint64 +type fe /*** ***/ [fpNumberOfLimbs]uint64 -// fe2 is element representation of 'fp2' which is quadratic extension of base field 'fp' +// fe2 is element representation of 'fp2' which is quadratic extention of base field 'fp' // Representation follows c[0] + c[1] * u encoding order. -type fe2 [2]fe +type fe2 /** ***/ [2]fe -// fe6 is element representation of 'fp6' field which is cubic extension of 'fp2' +// fe6 is element representation of 'fp6' field which is cubic extention of 'fp2' // Representation follows c[0] + c[1] * v + c[2] * v^2 encoding order. -type fe6 [3]fe2 +type fe6 /** ***/ [3]fe2 -// fe12 is element representation of 'fp12' field which is quadratic extension of 'fp6' +// fe12 is element representation of 'fp12' field which is quadratic extention of 'fp6' // Representation follows c[0] + c[1] * w encoding order. -type fe12 [2]fe6 +type fe12 /** ***/ [2]fe6 + +type wfe /*** ***/ [fpNumberOfLimbs * 2]uint64 +type wfe2 /** ***/ [2]wfe +type wfe6 /** ***/ [3]wfe2 func (fe *fe) setBytes(in []byte) *fe { - size := 48 l := len(in) - if l >= size { - l = size + if l >= fpByteSize { + l = fpByteSize } - padded := make([]byte, size) - copy(padded[size-l:], in[:]) + padded := make([]byte, fpByteSize) + copy(padded[fpByteSize-l:], in[:]) var a int - for i := 0; i < 6; i++ { - a = size - i*8 + for i := 0; i < fpNumberOfLimbs; i++ { + a = fpByteSize - i*8 fe[i] = uint64(padded[a-1]) | uint64(padded[a-2])<<8 | uint64(padded[a-3])<<16 | uint64(padded[a-4])<<24 | uint64(padded[a-5])<<32 | uint64(padded[a-6])<<40 | @@ -84,10 +71,10 @@ func (fe *fe) set(fe2 *fe) *fe { } func (fe *fe) bytes() []byte { - out := make([]byte, 48) + out := make([]byte, fpByteSize) var a int - for i := 0; i < 6; i++ { - a = 48 - i*8 + for i := 0; i < fpNumberOfLimbs; i++ { + a = fpByteSize - i*8 out[a-1] = byte(fe[i]) out[a-2] = byte(fe[i] >> 8) out[a-3] = byte(fe[i] >> 16) @@ -105,7 +92,7 @@ func (fe *fe) big() *big.Int { } func (fe *fe) string() (s string) { - for i := 5; i >= 0; i-- { + for i := fpNumberOfLimbs - 1; i >= 0; i-- { s = fmt.Sprintf("%s%16.16x", s, fe[i]) } return "0x" + s @@ -134,7 +121,7 @@ func (fe *fe) rand(r io.Reader) (*fe, error) { } func (fe *fe) isValid() bool { - return fe.cmp(&modulus) < 0 + return fe.cmp(&modulus) == -1 } func (fe *fe) isOdd() bool { @@ -156,7 +143,7 @@ func (fe *fe) isOne() bool { } func (fe *fe) cmp(fe2 *fe) int { - for i := 5; i >= 0; i-- { + for i := fpNumberOfLimbs - 1; i >= 0; i-- { if fe[i] > fe2[i] { return 1 } else if fe[i] < fe2[i] { @@ -170,30 +157,37 @@ func (fe *fe) equal(fe2 *fe) bool { return fe2[0] == fe[0] && fe2[1] == fe[1] && fe2[2] == fe[2] && fe2[3] == fe[3] && fe2[4] == fe[4] && fe2[5] == fe[5] } +func (e *fe) signBE() bool { + negZ, z := new(fe), new(fe) + fromMont(z, e) + neg(negZ, z) + return negZ.cmp(z) > -1 +} + func (e *fe) sign() bool { r := new(fe) fromMont(r, e) return r[0]&1 == 0 } -func (fe *fe) div2(e uint64) { - fe[0] = fe[0]>>1 | fe[1]<<63 - fe[1] = fe[1]>>1 | fe[2]<<63 - fe[2] = fe[2]>>1 | fe[3]<<63 - fe[3] = fe[3]>>1 | fe[4]<<63 - fe[4] = fe[4]>>1 | fe[5]<<63 - fe[5] = fe[5]>>1 | e<<63 +func (e *fe) div2(u uint64) { + e[0] = e[0]>>1 | e[1]<<63 + e[1] = e[1]>>1 | e[2]<<63 + e[2] = e[2]>>1 | e[3]<<63 + e[3] = e[3]>>1 | e[4]<<63 + e[4] = e[4]>>1 | e[5]<<63 + e[5] = e[5]>>1 | u<<63 } -func (fe *fe) mul2() uint64 { - e := fe[5] >> 63 - fe[5] = fe[5]<<1 | fe[4]>>63 - fe[4] = fe[4]<<1 | fe[3]>>63 - fe[3] = fe[3]<<1 | fe[2]>>63 - fe[2] = fe[2]<<1 | fe[1]>>63 - fe[1] = fe[1]<<1 | fe[0]>>63 - fe[0] = fe[0] << 1 - return e +func (e *fe) mul2() uint64 { + u := e[5] >> 63 + e[5] = e[5]<<1 | e[4]>>63 + e[4] = e[4]<<1 | e[3]>>63 + e[3] = e[3]<<1 | e[2]>>63 + e[2] = e[2]<<1 | e[1]>>63 + e[1] = e[1]<<1 | e[0]>>63 + e[0] = e[0] << 1 + return u } func (e *fe2) zero() *fe2 { @@ -214,16 +208,28 @@ func (e *fe2) set(e2 *fe2) *fe2 { return e } +func (e *fe2) fromMont(a *fe2) { + fromMont(&e[0], &a[0]) + fromMont(&e[1], &a[1]) +} + +func (e *fe2) fromWide(w *wfe2) { + fromWide(&e[0], &w[0]) + fromWide(&e[1], &w[1]) +} + func (e *fe2) rand(r io.Reader) (*fe2, error) { a0, err := new(fe).rand(r) if err != nil { return nil, err } + e[0].set(a0) a1, err := new(fe).rand(r) if err != nil { return nil, err } - return &fe2{*a0, *a1}, nil + e[1].set(a1) + return e, nil } func (e *fe2) isOne() bool { @@ -238,6 +244,13 @@ func (e *fe2) equal(e2 *fe2) bool { return e[0].equal(&e2[0]) && e[1].equal(&e2[1]) } +func (e *fe2) signBE() bool { + if !e[1].isZero() { + return e[1].signBE() + } + return e[0].signBE() +} + func (e *fe2) sign() bool { r := new(fe) if !e[0].isZero() { @@ -269,20 +282,35 @@ func (e *fe6) set(e2 *fe6) *fe6 { return e } +func (e *fe6) fromMont(a *fe6) { + e[0].fromMont(&a[0]) + e[1].fromMont(&a[1]) + e[2].fromMont(&a[2]) +} + +func (e *fe6) fromWide(w *wfe6) { + e[0].fromWide(&w[0]) + e[1].fromWide(&w[1]) + e[2].fromWide(&w[2]) +} + func (e *fe6) rand(r io.Reader) (*fe6, error) { a0, err := new(fe2).rand(r) if err != nil { return nil, err } + e[0].set(a0) a1, err := new(fe2).rand(r) if err != nil { return nil, err } + e[1].set(a1) a2, err := new(fe2).rand(r) if err != nil { return nil, err } - return &fe6{*a0, *a1, *a2}, nil + e[2].set(a2) + return e, nil } func (e *fe6) isOne() bool { @@ -315,16 +343,23 @@ func (e *fe12) set(e2 *fe12) *fe12 { return e } +func (e *fe12) fromMont(a *fe12) { + e[0].fromMont(&a[0]) + e[1].fromMont(&a[1]) +} + func (e *fe12) rand(r io.Reader) (*fe12, error) { a0, err := new(fe6).rand(r) if err != nil { return nil, err } + e[0].set(a0) a1, err := new(fe6).rand(r) if err != nil { return nil, err } - return &fe12{*a0, *a1}, nil + e[1].set(a1) + return e, nil } func (e *fe12) isOne() bool { @@ -338,3 +373,32 @@ func (e *fe12) isZero() bool { func (e *fe12) equal(e2 *fe12) bool { return e[0].equal(&e2[0]) && e[1].equal(&e2[1]) } + +func (fe *wfe) set(fe2 *wfe) *wfe { + fe[0] = fe2[0] + fe[1] = fe2[1] + fe[2] = fe2[2] + fe[3] = fe2[3] + fe[4] = fe2[4] + fe[5] = fe2[5] + fe[6] = fe2[6] + fe[7] = fe2[7] + fe[8] = fe2[8] + fe[9] = fe2[9] + fe[10] = fe2[10] + fe[11] = fe2[11] + return fe +} + +func (fe *wfe2) set(fe2 *wfe2) *wfe2 { + fe[0].set(&fe2[0]) + fe[1].set(&fe2[1]) + return fe +} + +func (fe *wfe6) set(fe2 *wfe6) *wfe6 { + fe[0].set(&fe2[0]) + fe[1].set(&fe2[1]) + fe[2].set(&fe2[2]) + return fe +} diff --git a/crypto/bls12381/field_element_test.go b/crypto/bls12381/field_element_test.go index 70bbe5cfe5..6bbad86959 100644 --- a/crypto/bls12381/field_element_test.go +++ b/crypto/bls12381/field_element_test.go @@ -8,6 +8,7 @@ import ( ) func TestFieldElementValidation(t *testing.T) { + // fe zero := new(fe).zero() if !zero.isValid() { t.Fatal("zero must be valid") @@ -59,8 +60,7 @@ func TestFieldElementEquality(t *testing.T) { t.Fatal("a == a") } b2 := new(fe2) - fp2 := newFp2() - fp2.add(b2, a2, one2) + fp2Add(b2, a2, one2) if a2.equal(b2) { t.Fatal("a != a + 1") } @@ -78,8 +78,7 @@ func TestFieldElementEquality(t *testing.T) { t.Fatal("a == a") } b6 := new(fe6) - fp6 := newFp6(fp2) - fp6.add(b6, a6, one6) + fp6Add(b6, a6, one6) if a6.equal(b6) { t.Fatal("a != a + 1") } @@ -97,11 +96,11 @@ func TestFieldElementEquality(t *testing.T) { t.Fatal("a == a") } b12 := new(fe12) - fp12 := newFp12(fp6) - fp12.add(b12, a12, one12) + fp12Add(b12, a12, one12) if a12.equal(b12) { t.Fatal("a != a + 1") } + } func TestFieldElementHelpers(t *testing.T) { @@ -159,13 +158,13 @@ func TestFieldElementHelpers(t *testing.T) { func TestFieldElementSerialization(t *testing.T) { t.Run("zero", func(t *testing.T) { - in := make([]byte, 48) + in := make([]byte, fpByteSize) fe := new(fe).setBytes(in) if !fe.isZero() { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } if !bytes.Equal(in, fe.bytes()) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } }) t.Run("bytes", func(t *testing.T) { @@ -173,7 +172,7 @@ func TestFieldElementSerialization(t *testing.T) { a, _ := new(fe).rand(rand.Reader) b := new(fe).setBytes(a.bytes()) if !a.equal(b) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } } }) @@ -182,7 +181,7 @@ func TestFieldElementSerialization(t *testing.T) { a, _ := new(fe).rand(rand.Reader) b := new(fe).setBig(a.big()) if !a.equal(b) { - t.Fatal("bad encoding or decoding") + t.Fatal("encoding or decoding failed") } } }) @@ -194,7 +193,7 @@ func TestFieldElementSerialization(t *testing.T) { t.Fatal(err) } if !a.equal(b) { - t.Fatal("bad encoding or decoding") + t.Fatal("encoding or decoding failed") } } }) @@ -205,24 +204,24 @@ func TestFieldElementByteInputs(t *testing.T) { in := make([]byte, 0) a := new(fe).setBytes(in) if !a.equal(zero) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } - in = make([]byte, 48) + in = make([]byte, fpByteSize) a = new(fe).setBytes(in) if !a.equal(zero) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } - in = make([]byte, 64) + in = make([]byte, fpByteSize+200) a = new(fe).setBytes(in) if !a.equal(zero) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } - in = make([]byte, 49) - in[47] = 1 + in = make([]byte, fpByteSize+1) + in[fpByteSize-1] = 1 normalOne := &fe{1, 0, 0, 0, 0, 0} a = new(fe).setBytes(in) if !a.equal(normalOne) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } } @@ -230,21 +229,21 @@ func TestFieldElementCopy(t *testing.T) { a, _ := new(fe).rand(rand.Reader) b := new(fe).set(a) if !a.equal(b) { - t.Fatal("bad copy, 1") + t.Fatal("copy failed") } a2, _ := new(fe2).rand(rand.Reader) b2 := new(fe2).set(a2) if !a2.equal(b2) { - t.Fatal("bad copy, 2") + t.Fatal("copy failed") } a6, _ := new(fe6).rand(rand.Reader) b6 := new(fe6).set(a6) if !a6.equal(b6) { - t.Fatal("bad copy, 6") + t.Fatal("copy failed") } a12, _ := new(fe12).rand(rand.Reader) b12 := new(fe12).set(a12) if !a12.equal(b12) { - t.Fatal("bad copy, 12") + t.Fatal("copy failed2") } } diff --git a/crypto/bls12381/fp.go b/crypto/bls12381/fp.go index 09f6f49bc0..a163a222da 100644 --- a/crypto/bls12381/fp.go +++ b/crypto/bls12381/fp.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( @@ -23,8 +7,8 @@ import ( func fromBytes(in []byte) (*fe, error) { fe := &fe{} - if len(in) != 48 { - return nil, errors.New("input string should be equal 48 bytes") + if len(in) != fpByteSize { + return nil, errors.New("input string must be equal 48 bytes") } fe.setBytes(in) if !fe.isValid() { @@ -34,6 +18,37 @@ func fromBytes(in []byte) (*fe, error) { return fe, nil } +func from64Bytes(in []byte) (*fe, error) { + if len(in) != 32*2 { + return nil, errors.New("input string must be equal 64 bytes") + } + a0 := make([]byte, fpByteSize) + copy(a0[fpByteSize-32:fpByteSize], in[:32]) + a1 := make([]byte, fpByteSize) + copy(a1[fpByteSize-32:fpByteSize], in[32:]) + e0, err := fromBytes(a0) + if err != nil { + return nil, err + } + e1, err := fromBytes(a1) + if err != nil { + return nil, err + } + // F = 2 ^ 256 * R + F := fe{ + 0x75b3cd7c5ce820f, + 0x3ec6ba621c3edb0b, + 0x168a13d82bff6bce, + 0x87663c4bf8c449d2, + 0x15f34c83ddc8d830, + 0xf9628b49caa2e85, + } + + mul(e0, e0, &F) + add(e1, e1, e0) + return e1, nil +} + func fromBig(in *big.Int) (*fe, error) { fe := new(fe).setBig(in) if !fe.isValid() { @@ -81,6 +96,28 @@ func fromMont(c, a *fe) { mul(c, a, &fe{1}) } +func wfp2MulGeneric(c *wfe2, a, b *fe2) { + wt0, wt1 := new(wfe), new(wfe) + t0, t1 := new(fe), new(fe) + wmul(wt0, &a[0], &b[0]) + wmul(wt1, &a[1], &b[1]) + wsub(&c[0], wt0, wt1) + lwaddAssign(wt0, wt1) + ladd(t0, &a[0], &a[1]) + ladd(t1, &b[0], &b[1]) + wmul(wt1, t0, t1) + lwsub(&c[1], wt1, wt0) +} + +func wfp2SquareGeneric(c *wfe2, a *fe2) { + t0, t1, t2 := new(fe), new(fe), new(fe) + ladd(t0, &a[0], &a[1]) + sub(t1, &a[0], &a[1]) + ldouble(t2, &a[0]) + wmul(&c[0], t1, t0) + wmul(&c[1], t2, &a[1]) +} + func exp(c, a *fe, e *big.Int) { z := new(fe).set(r1) for i := e.BitLen(); i >= 0; i-- { @@ -105,7 +142,7 @@ func inverse(inv, e *fe) { var z uint64 var found = false // Phase 1 - for i := 0; i < 768; i++ { + for i := 0; i < sixWordBitSize*2; i++ { if v.isZero() { found = true break @@ -135,7 +172,7 @@ func inverse(inv, e *fe) { return } - if k < 381 || k > 381+384 { + if k < fpBitSize || k > fpBitSize+sixWordBitSize { inv.zero() return } @@ -147,21 +184,189 @@ func inverse(inv, e *fe) { lsubAssign(u, r) // Phase 2 - for i := k; i < 384*2; i++ { + for i := k; i < 2*sixWordBitSize; i++ { double(u, u) } inv.set(u) } +func inverseBatch(in []fe) { + + n, N, setFirst := 0, len(in), false + + for i := 0; i < len(in); i++ { + if !in[i].isZero() { + n++ + } + } + if n == 0 { + return + } + + tA := make([]fe, n) + tB := make([]fe, n) + + for i, j := 0, 0; i < N; i++ { + if !in[i].isZero() { + if !setFirst { + setFirst = true + tA[j].set(&in[i]) + } else { + mul(&tA[j], &in[i], &tA[j-1]) + } + j = j + 1 + } + } + + inverse(&tB[n-1], &tA[n-1]) + for i, j := N-1, n-1; j != 0; i-- { + if !in[i].isZero() { + mul(&tB[j-1], &tB[j], &in[i]) + j = j - 1 + } + } + + for i, j := 0, 0; i < N; i++ { + if !in[i].isZero() { + if setFirst { + setFirst = false + in[i].set(&tB[j]) + } else { + mul(&in[i], &tA[j-1], &tB[j]) + } + j = j + 1 + } + } +} + +func rsqrt(c, a *fe) bool { + t0, t1 := new(fe), new(fe) + sqrtAddchain(t0, a) + mul(t1, t0, a) + square(t1, t1) + ret := t1.equal(a) + c.set(t0) + return ret +} + func sqrt(c, a *fe) bool { + u, v := new(fe).set(a), new(fe) + // a ^ (p - 3) / 4 + sqrtAddchain(c, a) + // a ^ (p + 1) / 4 + mul(c, c, u) + + square(v, c) + return u.equal(v) +} + +func _sqrt(c, a *fe) bool { u, v := new(fe).set(a), new(fe) exp(c, a, pPlus1Over4) square(v, c) return u.equal(v) } -func isQuadraticNonResidue(elem *fe) bool { - result := new(fe) - exp(result, elem, pMinus1Over2) - return !result.isOne() +func sqrtAddchain(c, a *fe) { + chain := func(c *fe, n int, a *fe) { + for i := 0; i < n; i++ { + square(c, c) + } + mul(c, c, a) + } + + t := make([]fe, 16) + t[13].set(a) + square(&t[0], &t[13]) + mul(&t[8], &t[0], &t[13]) + square(&t[4], &t[0]) + mul(&t[1], &t[8], &t[0]) + mul(&t[6], &t[4], &t[8]) + mul(&t[9], &t[1], &t[4]) + mul(&t[12], &t[6], &t[4]) + mul(&t[3], &t[9], &t[4]) + mul(&t[7], &t[12], &t[4]) + mul(&t[15], &t[3], &t[4]) + mul(&t[10], &t[7], &t[4]) + mul(&t[2], &t[15], &t[4]) + mul(&t[11], &t[10], &t[4]) + square(&t[0], &t[3]) + mul(&t[14], &t[11], &t[4]) + mul(&t[5], &t[0], &t[8]) + mul(&t[4], &t[0], &t[1]) + + chain(&t[0], 12, &t[15]) + chain(&t[0], 7, &t[7]) + chain(&t[0], 4, &t[1]) + chain(&t[0], 6, &t[6]) + chain(&t[0], 7, &t[11]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 2, &t[8]) + chain(&t[0], 6, &t[3]) + chain(&t[0], 6, &t[3]) + chain(&t[0], 6, &t[9]) + chain(&t[0], 3, &t[8]) + chain(&t[0], 7, &t[3]) + chain(&t[0], 4, &t[3]) + chain(&t[0], 6, &t[7]) + chain(&t[0], 6, &t[14]) + chain(&t[0], 3, &t[13]) + chain(&t[0], 8, &t[3]) + chain(&t[0], 7, &t[11]) + chain(&t[0], 5, &t[12]) + chain(&t[0], 6, &t[3]) + chain(&t[0], 6, &t[5]) + chain(&t[0], 4, &t[9]) + chain(&t[0], 8, &t[5]) + chain(&t[0], 4, &t[3]) + chain(&t[0], 7, &t[11]) + chain(&t[0], 9, &t[10]) + chain(&t[0], 2, &t[8]) + chain(&t[0], 5, &t[6]) + chain(&t[0], 7, &t[1]) + chain(&t[0], 7, &t[9]) + chain(&t[0], 6, &t[11]) + chain(&t[0], 5, &t[5]) + chain(&t[0], 5, &t[10]) + chain(&t[0], 5, &t[10]) + chain(&t[0], 8, &t[3]) + chain(&t[0], 7, &t[2]) + chain(&t[0], 9, &t[7]) + chain(&t[0], 5, &t[3]) + chain(&t[0], 3, &t[8]) + chain(&t[0], 8, &t[7]) + chain(&t[0], 3, &t[8]) + chain(&t[0], 7, &t[9]) + chain(&t[0], 9, &t[7]) + chain(&t[0], 6, &t[2]) + chain(&t[0], 6, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 4, &t[3]) + chain(&t[0], 3, &t[8]) + chain(&t[0], 8, &t[2]) + chain(&t[0], 7, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 4, &t[7]) + chain(&t[0], 4, &t[6]) + chain(&t[0], 7, &t[4]) + chain(&t[0], 5, &t[5]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 5, &t[4]) + chain(&t[0], 4, &t[3]) + chain(&t[0], 6, &t[2]) + chain(&t[0], 4, &t[1]) + square(c, &t[0]) +} + +func isQuadraticNonResidue(a *fe) bool { + if a.isZero() { + return true + } + return !sqrt(new(fe), a) } diff --git a/crypto/bls12381/fp12.go b/crypto/bls12381/fp12.go index 51e949fe5f..8b5af825ce 100644 --- a/crypto/bls12381/fp12.go +++ b/crypto/bls12381/fp12.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( @@ -27,21 +11,30 @@ type fp12 struct { } type fp12temp struct { - t2 [9]*fe2 - t6 [5]*fe6 - t12 *fe12 + t2 [7]*fe2 + t6 [4]*fe6 + wt2 [3]*wfe2 + wt6 [3]*wfe6 } func newFp12Temp() fp12temp { - t2 := [9]*fe2{} - t6 := [5]*fe6{} + t2 := [7]*fe2{} + t6 := [4]*fe6{} for i := 0; i < len(t2); i++ { t2[i] = &fe2{} } for i := 0; i < len(t6); i++ { t6[i] = &fe6{} } - return fp12temp{t2, t6, &fe12{}} + wt2 := [3]*wfe2{} + for i := 0; i < len(wt2); i++ { + wt2[i] = &wfe2{} + } + wt6 := [3]*wfe6{} + for i := 0; i < len(wt6); i++ { + wt6[i] = &wfe6{} + } + return fp12temp{t2, t6, wt2, wt6} } func newFp12(fp6 *fp6) *fp12 { @@ -58,14 +51,14 @@ func (e *fp12) fp2() *fp2 { func (e *fp12) fromBytes(in []byte) (*fe12, error) { if len(in) != 576 { - return nil, errors.New("input string should be larger than 96 bytes") + return nil, errors.New("input string length must be equal to 576 bytes") } fp6 := e.fp6 - c1, err := fp6.fromBytes(in[:288]) + c1, err := fp6.fromBytes(in[:6*fpByteSize]) if err != nil { return nil, err } - c0, err := fp6.fromBytes(in[288:]) + c0, err := fp6.fromBytes(in[6*fpByteSize:]) if err != nil { return nil, err } @@ -74,9 +67,9 @@ func (e *fp12) fromBytes(in []byte) (*fe12, error) { func (e *fp12) toBytes(a *fe12) []byte { fp6 := e.fp6 - out := make([]byte, 576) - copy(out[:288], fp6.toBytes(&a[1])) - copy(out[288:], fp6.toBytes(&a[0])) + out := make([]byte, 12*fpByteSize) + copy(out[:6*fpByteSize], fp6.toBytes(&a[1])) + copy(out[6*fpByteSize:], fp6.toBytes(&a[0])) return out } @@ -92,138 +85,125 @@ func (e *fp12) one() *fe12 { return new(fe12).one() } -func (e *fp12) add(c, a, b *fe12) { - fp6 := e.fp6 - fp6.add(&c[0], &a[0], &b[0]) - fp6.add(&c[1], &a[1], &b[1]) +func fp12Add(c, a, b *fe12) { + fp6Add(&c[0], &a[0], &b[0]) + fp6Add(&c[1], &a[1], &b[1]) } -func (e *fp12) double(c, a *fe12) { - fp6 := e.fp6 - fp6.double(&c[0], &a[0]) - fp6.double(&c[1], &a[1]) +func fp12Double(c, a *fe12) { + fp6Double(&c[0], &a[0]) + fp6Double(&c[1], &a[1]) } -func (e *fp12) sub(c, a, b *fe12) { - fp6 := e.fp6 - fp6.sub(&c[0], &a[0], &b[0]) - fp6.sub(&c[1], &a[1], &b[1]) +func fp12Sub(c, a, b *fe12) { + fp6Sub(&c[0], &a[0], &b[0]) + fp6Sub(&c[1], &a[1], &b[1]) + } -func (e *fp12) neg(c, a *fe12) { - fp6 := e.fp6 - fp6.neg(&c[0], &a[0]) - fp6.neg(&c[1], &a[1]) +func fp12Neg(c, a *fe12) { + fp6Neg(&c[0], &a[0]) + fp6Neg(&c[1], &a[1]) } -func (e *fp12) conjugate(c, a *fe12) { - fp6 := e.fp6 +func fp12Conjugate(c, a *fe12) { c[0].set(&a[0]) - fp6.neg(&c[1], &a[1]) -} - -func (e *fp12) square(c, a *fe12) { - fp6, t := e.fp6, e.t6 - fp6.add(t[0], &a[0], &a[1]) - fp6.mul(t[2], &a[0], &a[1]) - fp6.mulByNonResidue(t[1], &a[1]) - fp6.addAssign(t[1], &a[0]) - fp6.mulByNonResidue(t[3], t[2]) - fp6.mulAssign(t[0], t[1]) - fp6.subAssign(t[0], t[2]) - fp6.sub(&c[0], t[0], t[3]) - fp6.double(&c[1], t[2]) -} - -func (e *fp12) cyclotomicSquare(c, a *fe12) { - t, fp2 := e.t2, e.fp2() - e.fp4Square(t[3], t[4], &a[0][0], &a[1][1]) - fp2.sub(t[2], t[3], &a[0][0]) - fp2.doubleAssign(t[2]) - fp2.add(&c[0][0], t[2], t[3]) - fp2.add(t[2], t[4], &a[1][1]) - fp2.doubleAssign(t[2]) - fp2.add(&c[1][1], t[2], t[4]) - e.fp4Square(t[3], t[4], &a[1][0], &a[0][2]) - e.fp4Square(t[5], t[6], &a[0][1], &a[1][2]) - fp2.sub(t[2], t[3], &a[0][1]) - fp2.doubleAssign(t[2]) - fp2.add(&c[0][1], t[2], t[3]) - fp2.add(t[2], t[4], &a[1][2]) - fp2.doubleAssign(t[2]) - fp2.add(&c[1][2], t[2], t[4]) - fp2.mulByNonResidue(t[3], t[6]) - fp2.add(t[2], t[3], &a[1][0]) - fp2.doubleAssign(t[2]) - fp2.add(&c[1][0], t[2], t[3]) - fp2.sub(t[2], t[5], &a[0][2]) - fp2.doubleAssign(t[2]) - fp2.add(&c[0][2], t[2], t[5]) + fp6Neg(&c[1], &a[1]) } func (e *fp12) mul(c, a, b *fe12) { - t, fp6 := e.t6, e.fp6 - fp6.mul(t[1], &a[0], &b[0]) - fp6.mul(t[2], &a[1], &b[1]) - fp6.add(t[0], t[1], t[2]) - fp6.mulByNonResidue(t[2], t[2]) - fp6.add(t[3], t[1], t[2]) - fp6.add(t[1], &a[0], &a[1]) - fp6.add(t[2], &b[0], &b[1]) - fp6.mulAssign(t[1], t[2]) - c[0].set(t[3]) - fp6.sub(&c[1], t[1], t[0]) + wt, t := e.wt6, e.t6 + e.fp6.wmul(wt[1], &a[0], &b[0]) + e.fp6.wmul(wt[2], &a[1], &b[1]) + fp6Add(t[0], &a[0], &a[1]) + fp6Add(t[3], &b[0], &b[1]) + e.fp6.wmul(wt[0], t[0], t[3]) + wfp6SubAssign(wt[0], wt[1]) + wfp6SubAssign(wt[0], wt[2]) + c[1].fromWide(wt[0]) + e.fp6.wmulByNonResidueAssign(wt[2]) + wfp6AddAssign(wt[1], wt[2]) + c[0].fromWide(wt[1]) + } func (e *fp12) mulAssign(a, b *fe12) { - t, fp6 := e.t6, e.fp6 - fp6.mul(t[1], &a[0], &b[0]) - fp6.mul(t[2], &a[1], &b[1]) - fp6.add(t[0], t[1], t[2]) - fp6.mulByNonResidue(t[2], t[2]) - fp6.add(t[3], t[1], t[2]) - fp6.add(t[1], &a[0], &a[1]) - fp6.add(t[2], &b[0], &b[1]) - fp6.mulAssign(t[1], t[2]) - a[0].set(t[3]) - fp6.sub(&a[1], t[1], t[0]) + wt, t := e.wt6, e.t6 + e.fp6.wmul(wt[1], &a[0], &b[0]) + e.fp6.wmul(wt[2], &a[1], &b[1]) + fp6Add(t[0], &a[0], &a[1]) + fp6Add(t[3], &b[0], &b[1]) + e.fp6.wmul(wt[0], t[0], t[3]) + wfp6SubAssign(wt[0], wt[1]) + wfp6SubAssign(wt[0], wt[2]) + a[1].fromWide(wt[0]) + e.fp6.wmulByNonResidueAssign(wt[2]) + wfp6AddAssign(wt[1], wt[2]) + a[0].fromWide(wt[1]) } -func (e *fp12) fp4Square(c0, c1, a0, a1 *fe2) { - t, fp2 := e.t2, e.fp2() - fp2.square(t[0], a0) - fp2.square(t[1], a1) - fp2.mulByNonResidue(t[2], t[1]) - fp2.add(c0, t[2], t[0]) - fp2.add(t[2], a0, a1) - fp2.squareAssign(t[2]) - fp2.subAssign(t[2], t[0]) - fp2.sub(c1, t[2], t[1]) +func (e *fp12) mul014(a *fe12, b0, b1, b4 *fe2) { + wt, t := e.wt6, e.t6 + e.fp6.wmul01(wt[0], &a[0], b0, b1) + e.fp6.wmul1(wt[1], &a[1], b4) + fp2LaddAssign(b1, b4) + fp6Ladd(t[2], &a[1], &a[0]) + e.fp6.wmul01(wt[2], t[2], b0, b1) + wfp6SubAssign(wt[2], wt[0]) + wfp6SubAssign(wt[2], wt[1]) + a[1].fromWide(wt[2]) + e.fp6.wmulByNonResidueAssign(wt[1]) + wfp6AddAssign(wt[0], wt[1]) + a[0].fromWide(wt[0]) +} + +func (e *fp12) square(c, a *fe12) { + t := e.t6 + // Multiplication and Squaring on Pairing-Friendly Fields + // Complex squaring algorithm + // https://eprint.iacr.org/2006/471 + + fp6Add(t[0], &a[0], &a[1]) + e.fp6.mul(t[2], &a[0], &a[1]) + e.fp6.mulByNonResidue(t[1], &a[1]) + fp6AddAssign(t[1], &a[0]) + e.fp6.mulByNonResidue(t[3], t[2]) + e.fp6.mul(t[0], t[0], t[1]) + fp6SubAssign(t[0], t[2]) + fp6Sub(&c[0], t[0], t[3]) + fp6Double(&c[1], t[2]) +} + +func (e *fp12) squareAssign(a *fe12) { + t := e.t6 + // Multiplication and Squaring on Pairing-Friendly Fields + // Complex squaring algorithm + // https://eprint.iacr.org/2006/471 + + fp6Add(t[0], &a[0], &a[1]) + e.fp6.mul(t[2], &a[0], &a[1]) + e.fp6.mulByNonResidue(t[1], &a[1]) + fp6AddAssign(t[1], &a[0]) + e.fp6.mulByNonResidue(t[3], t[2]) + e.fp6.mul(t[0], t[0], t[1]) + fp6SubAssign(t[0], t[2]) + fp6Sub(&a[0], t[0], t[3]) + fp6Double(&a[1], t[2]) } func (e *fp12) inverse(c, a *fe12) { - fp6, t := e.fp6, e.t6 - fp6.square(t[0], &a[0]) - fp6.square(t[1], &a[1]) - fp6.mulByNonResidue(t[1], t[1]) - fp6.sub(t[1], t[0], t[1]) - fp6.inverse(t[0], t[1]) - fp6.mul(&c[0], &a[0], t[0]) - fp6.mulAssign(t[0], &a[1]) - fp6.neg(&c[1], t[0]) -} + // Guide to Pairing Based Cryptography + // Algorithm 5.16 -func (e *fp12) mulBy014Assign(a *fe12, c0, c1, c4 *fe2) { - fp2, fp6, t, t2 := e.fp2(), e.fp6, e.t6, e.t2[0] - fp6.mulBy01(t[0], &a[0], c0, c1) - fp6.mulBy1(t[1], &a[1], c4) - fp2.add(t2, c1, c4) - fp6.add(t[2], &a[1], &a[0]) - fp6.mulBy01Assign(t[2], c0, t2) - fp6.subAssign(t[2], t[0]) - fp6.sub(&a[1], t[2], t[1]) - fp6.mulByNonResidue(t[1], t[1]) - fp6.add(&a[0], t[1], t[0]) + t := e.t6 + e.fp6.square(t[0], &a[0]) // a0^2 + e.fp6.square(t[1], &a[1]) // a1^2 + e.fp6.mulByNonResidue(t[1], t[1]) // βa1^2 + fp6SubAssign(t[0], t[1]) // v = (a0^2 - a1^2) + e.fp6.inverse(t[1], t[0]) // v = v^-1 + e.fp6.mul(&c[0], &a[0], t[1]) // c0 = a0v + e.fp6.mulAssign(t[1], &a[1]) // + fp6Neg(&c[1], t[1]) // c1 = -a1v } func (e *fp12) exp(c, a *fe12, s *big.Int) { @@ -240,7 +220,7 @@ func (e *fp12) exp(c, a *fe12, s *big.Int) { func (e *fp12) cyclotomicExp(c, a *fe12, s *big.Int) { z := e.one() for i := s.BitLen() - 1; i >= 0; i-- { - e.cyclotomicSquare(z, z) + e.cyclotomicSquare(z) if s.Bit(i) == 1 { e.mul(z, z, a) } @@ -248,30 +228,76 @@ func (e *fp12) cyclotomicExp(c, a *fe12, s *big.Int) { c.set(z) } -func (e *fp12) frobeniusMap(c, a *fe12, power uint) { - fp6 := e.fp6 - fp6.frobeniusMap(&c[0], &a[0], power) - fp6.frobeniusMap(&c[1], &a[1], power) - switch power { - case 0: - return - case 6: - fp6.neg(&c[1], &c[1]) - default: - fp6.mulByBaseField(&c[1], &c[1], &frobeniusCoeffs12[power]) - } +func (e *fp12) cyclotomicSquare(a *fe12) { + t := e.t2 + // Guide to Pairing Based Cryptography + // 5.5.4 Airthmetic in Cyclotomic Groups + + e.fp4Square(t[3], t[4], &a[0][0], &a[1][1]) + fp2Sub(t[2], t[3], &a[0][0]) + fp2DoubleAssign(t[2]) + fp2Add(&a[0][0], t[2], t[3]) + fp2Add(t[2], t[4], &a[1][1]) + fp2DoubleAssign(t[2]) + fp2Add(&a[1][1], t[2], t[4]) + e.fp4Square(t[3], t[4], &a[1][0], &a[0][2]) + e.fp4Square(t[5], t[6], &a[0][1], &a[1][2]) + fp2Sub(t[2], t[3], &a[0][1]) + fp2DoubleAssign(t[2]) + fp2Add(&a[0][1], t[2], t[3]) + fp2Add(t[2], t[4], &a[1][2]) + fp2DoubleAssign(t[2]) + fp2Add(&a[1][2], t[2], t[4]) + mulByNonResidue(t[3], t[6]) + fp2Add(t[2], t[3], &a[1][0]) + fp2DoubleAssign(t[2]) + fp2Add(&a[1][0], t[2], t[3]) + fp2Sub(t[2], t[5], &a[0][2]) + fp2DoubleAssign(t[2]) + fp2Add(&a[0][2], t[2], t[5]) } -func (e *fp12) frobeniusMapAssign(a *fe12, power uint) { - fp6 := e.fp6 - fp6.frobeniusMapAssign(&a[0], power) - fp6.frobeniusMapAssign(&a[1], power) - switch power { - case 0: - return - case 6: - fp6.neg(&a[1], &a[1]) - default: - fp6.mulByBaseField(&a[1], &a[1], &frobeniusCoeffs12[power]) - } +func (e *fp12) fp4Square(c0, c1, a0, a1 *fe2) { + wt, t := e.wt2, e.t2 + // Multiplication and Squaring on Pairing-Friendly Fields + // Karatsuba squaring algorithm + // https://eprint.iacr.org/2006/471 + + wfp2Square(wt[0], a0) + wfp2Square(wt[1], a1) + wfp2MulByNonResidue(wt[2], wt[1]) + wfp2AddAssign(wt[2], wt[0]) + c0.fromWide(wt[2]) + fp2Add(t[0], a0, a1) + wfp2Square(wt[2], t[0]) + wfp2SubAssign(wt[2], wt[0]) + wfp2SubAssign(wt[2], wt[1]) + c1.fromWide(wt[2]) +} + +func (e *fp12) frobeniusMap1(a *fe12) { + fp6, fp2 := e.fp6, e.fp6.fp2 + fp6.frobeniusMap1(&a[0]) + fp6.frobeniusMap1(&a[1]) + fp2.mulAssign(&a[1][0], &frobeniusCoeffs12[1]) + fp2.mulAssign(&a[1][1], &frobeniusCoeffs12[1]) + fp2.mulAssign(&a[1][2], &frobeniusCoeffs12[1]) +} + +func (e *fp12) frobeniusMap2(a *fe12) { + fp6, fp2 := e.fp6, e.fp6.fp2 + fp6.frobeniusMap2(&a[0]) + fp6.frobeniusMap2(&a[1]) + fp2.mulAssign(&a[1][0], &frobeniusCoeffs12[2]) + fp2.mulAssign(&a[1][1], &frobeniusCoeffs12[2]) + fp2.mulAssign(&a[1][2], &frobeniusCoeffs12[2]) +} + +func (e *fp12) frobeniusMap3(a *fe12) { + fp6, fp2 := e.fp6, e.fp6.fp2 + fp6.frobeniusMap3(&a[0]) + fp6.frobeniusMap3(&a[1]) + fp2.mulAssign(&a[1][0], &frobeniusCoeffs12[3]) + fp2.mulAssign(&a[1][1], &frobeniusCoeffs12[3]) + fp2.mulAssign(&a[1][2], &frobeniusCoeffs12[3]) } diff --git a/crypto/bls12381/fp2.go b/crypto/bls12381/fp2.go index 0f1c5a23ac..aad59d32b3 100644 --- a/crypto/bls12381/fp2.go +++ b/crypto/bls12381/fp2.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( @@ -22,7 +6,8 @@ import ( ) type fp2Temp struct { - t [4]*fe + t [3]*fe + w *wfe2 } type fp2 struct { @@ -30,11 +15,11 @@ type fp2 struct { } func newFp2Temp() fp2Temp { - t := [4]*fe{} + t := [3]*fe{} for i := 0; i < len(t); i++ { t[i] = &fe{} } - return fp2Temp{t} + return fp2Temp{t, &wfe2{}} } func newFp2() *fp2 { @@ -43,14 +28,14 @@ func newFp2() *fp2 { } func (e *fp2) fromBytes(in []byte) (*fe2, error) { - if len(in) != 96 { - return nil, errors.New("length of input string should be 96 bytes") + if len(in) != 2*fpByteSize { + return nil, errors.New("input string must be equal to 96 bytes") } - c1, err := fromBytes(in[:48]) + c1, err := fromBytes(in[:fpByteSize]) if err != nil { return nil, err } - c0, err := fromBytes(in[48:]) + c0, err := fromBytes(in[fpByteSize:]) if err != nil { return nil, err } @@ -58,9 +43,9 @@ func (e *fp2) fromBytes(in []byte) (*fe2, error) { } func (e *fp2) toBytes(a *fe2) []byte { - out := make([]byte, 96) - copy(out[:48], toBytes(&a[1])) - copy(out[48:], toBytes(&a[0])) + out := make([]byte, 2*fpByteSize) + copy(out[:fpByteSize], toBytes(&a[1])) + copy(out[fpByteSize:], toBytes(&a[0])) return out } @@ -76,82 +61,36 @@ func (e *fp2) one() *fe2 { return new(fe2).one() } -func (e *fp2) add(c, a, b *fe2) { - add(&c[0], &a[0], &b[0]) - add(&c[1], &a[1], &b[1]) -} - -func (e *fp2) addAssign(a, b *fe2) { - addAssign(&a[0], &b[0]) - addAssign(&a[1], &b[1]) -} - -func (e *fp2) ladd(c, a, b *fe2) { - ladd(&c[0], &a[0], &b[0]) - ladd(&c[1], &a[1], &b[1]) -} - -func (e *fp2) double(c, a *fe2) { - double(&c[0], &a[0]) - double(&c[1], &a[1]) -} - -func (e *fp2) doubleAssign(a *fe2) { - doubleAssign(&a[0]) - doubleAssign(&a[1]) -} - -func (e *fp2) ldouble(c, a *fe2) { - ldouble(&c[0], &a[0]) - ldouble(&c[1], &a[1]) -} - -func (e *fp2) sub(c, a, b *fe2) { - sub(&c[0], &a[0], &b[0]) - sub(&c[1], &a[1], &b[1]) -} - -func (e *fp2) subAssign(c, a *fe2) { - subAssign(&c[0], &a[0]) - subAssign(&c[1], &a[1]) -} - -func (e *fp2) neg(c, a *fe2) { +func fp2Neg(c, a *fe2) { neg(&c[0], &a[0]) neg(&c[1], &a[1]) } +func fp2Conjugate(c, a *fe2) { + c[0].set(&a[0]) + neg(&c[1], &a[1]) +} + func (e *fp2) mul(c, a, b *fe2) { - t := e.t - mul(t[1], &a[0], &b[0]) - mul(t[2], &a[1], &b[1]) - add(t[0], &a[0], &a[1]) - add(t[3], &b[0], &b[1]) - sub(&c[0], t[1], t[2]) - addAssign(t[1], t[2]) - mul(t[0], t[0], t[3]) - sub(&c[1], t[0], t[1]) + wfp2Mul(e.w, b, a) + c.fromWide(e.w) } func (e *fp2) mulAssign(a, b *fe2) { - t := e.t - mul(t[1], &a[0], &b[0]) - mul(t[2], &a[1], &b[1]) - add(t[0], &a[0], &a[1]) - add(t[3], &b[0], &b[1]) - sub(&a[0], t[1], t[2]) - addAssign(t[1], t[2]) - mul(t[0], t[0], t[3]) - sub(&a[1], t[0], t[1]) + wfp2Mul(e.w, b, a) + a.fromWide(e.w) } func (e *fp2) square(c, a *fe2) { t := e.t - ladd(t[0], &a[0], &a[1]) - sub(t[1], &a[0], &a[1]) - ldouble(t[2], &a[0]) - mul(&c[0], t[0], t[1]) - mul(&c[1], t[2], &a[1]) + // Guide to Pairing Based Cryptography + // Algorithm 5.16 + + ladd(t[0], &a[0], &a[1]) // (a0 + a1) + sub(t[1], &a[0], &a[1]) // (a0 - a1) + ldouble(t[2], &a[0]) // 2a0 + mul(&c[0], t[0], t[1]) // c0 = (a0 + a1)(a0 - a1) + mul(&c[1], t[2], &a[1]) // c1 = 2a0a1 } func (e *fp2) squareAssign(a *fe2) { @@ -163,18 +102,23 @@ func (e *fp2) squareAssign(a *fe2) { mul(&a[1], t[2], &a[1]) } -func (e *fp2) mulByNonResidue(c, a *fe2) { - t := e.t - sub(t[0], &a[0], &a[1]) - add(&c[1], &a[0], &a[1]) - c[0].set(t[0]) +func (e *fp2) mul0(c, a *fe2, b *fe) { + mul(&c[0], &a[0], b) + mul(&c[1], &a[1], b) +} + +func (e *fp2) mul0Assign(a *fe2, b *fe) { + mul(&a[0], &a[0], b) + mul(&a[1], &a[1], b) } func (e *fp2) mulByB(c, a *fe2) { t := e.t + // c0 = 4a0 - 4a1 + // c1 = 4a0 + 4a1 double(t[0], &a[0]) - double(t[1], &a[1]) doubleAssign(t[0]) + double(t[1], &a[1]) doubleAssign(t[1]) sub(&c[0], t[0], t[1]) add(&c[1], t[0], t[1]) @@ -182,18 +126,70 @@ func (e *fp2) mulByB(c, a *fe2) { func (e *fp2) inverse(c, a *fe2) { t := e.t - square(t[0], &a[0]) - square(t[1], &a[1]) - addAssign(t[0], t[1]) - inverse(t[0], t[0]) - mul(&c[0], &a[0], t[0]) - mul(t[0], t[0], &a[1]) - neg(&c[1], t[0]) + // Guide to Pairing Based Cryptography + // Algorithm 5.16 + + square(t[0], &a[0]) // a0^2 + square(t[1], &a[1]) // a1^2 + addAssign(t[0], t[1]) // a0^2 + a1^2 + inverse(t[0], t[0]) // (a0^2 + a1^2)^-1 + mul(&c[0], &a[0], t[0]) // c0 = a0(a0^2 + a1^2)^-1 + mul(t[0], t[0], &a[1]) // a1(a0^2 + a1^2)^-1 + neg(&c[1], t[0]) // c1 = a1(a0^2 + a1^2)^-1 } -func (e *fp2) mulByFq(c, a *fe2, b *fe) { - mul(&c[0], &a[0], b) - mul(&c[1], &a[1], b) +func (e *fp2) inverseBatch(in []fe2) { + + n, N, setFirst := 0, len(in), false + + for i := 0; i < len(in); i++ { + if !in[i].isZero() { + n++ + } + } + if n == 0 { + return + } + + tA := make([]fe2, n) + tB := make([]fe2, n) + + // a, ab, abc, abcd, ... + for i, j := 0, 0; i < N; i++ { + if !in[i].isZero() { + if !setFirst { + setFirst = true + tA[j].set(&in[i]) + } else { + e.mul(&tA[j], &in[i], &tA[j-1]) + } + j = j + 1 + } + } + + // (abcd...)^-1 + e.inverse(&tB[n-1], &tA[n-1]) + + // a^-1, ab^-1, abc^-1, abcd^-1, ... + for i, j := N-1, n-1; j != 0; i-- { + if !in[i].isZero() { + e.mul(&tB[j-1], &tB[j], &in[i]) + j = j - 1 + } + } + + // a^-1, b^-1, c^-1, d^-1 + for i, j := 0, 0; i < N; i++ { + if !in[i].isZero() { + if setFirst { + setFirst = false + in[i].set(&tB[j]) + } else { + e.mul(&in[i], &tA[j-1], &tB[j]) + } + j = j + 1 + } + } } func (e *fp2) exp(c, a *fe2, s *big.Int) { @@ -207,19 +203,13 @@ func (e *fp2) exp(c, a *fe2, s *big.Int) { c.set(z) } -func (e *fp2) frobeniusMap(c, a *fe2, power uint) { - c[0].set(&a[0]) - if power%2 == 1 { - neg(&c[1], &a[1]) - return - } - c[1].set(&a[1]) +func (e *fp2) frobeniusMap1(a *fe2) { + fp2Conjugate(a, a) } -func (e *fp2) frobeniusMapAssign(a *fe2, power uint) { - if power%2 == 1 { - neg(&a[1], &a[1]) - return +func (e *fp2) frobeniusMap(a *fe2, power int) { + if power&1 == 1 { + fp2Conjugate(a, a) } } @@ -235,7 +225,7 @@ func (e *fp2) sqrt(c, a *fe2) bool { c[1].set(&x0[0]) return true } - e.add(alpha, alpha, e.one()) + fp2Add(alpha, alpha, e.one()) e.exp(alpha, alpha, pMinus1Over2) e.mul(c, alpha, x0) e.square(alpha, c) @@ -243,10 +233,74 @@ func (e *fp2) sqrt(c, a *fe2) bool { } func (e *fp2) isQuadraticNonResidue(a *fe2) bool { - // https://github.com/leovt/constructible/wiki/Taking-Square-Roots-in-quadratic-extension-Fields c0, c1 := new(fe), new(fe) square(c0, &a[0]) square(c1, &a[1]) add(c1, c1, c0) return isQuadraticNonResidue(c1) } + +// faster square root algorith is adapted from blst library +// https://github.com/supranational/blst/blob/master/src/sqrt.c + +func (e *fp2) sqrtBLST(out, inp *fe2) bool { + aa, bb := new(fe), new(fe) + ret := new(fe2) + square(aa, &inp[0]) + square(bb, &inp[1]) + add(aa, aa, bb) + sqrt(aa, aa) + sub(bb, &inp[0], aa) + add(aa, &inp[0], aa) + if aa.isZero() { + aa.set(bb) + } + mul(aa, aa, twoInv) + rsqrt(&ret[0], aa) + ret[1].set(&inp[1]) + mul(&ret[1], &ret[1], twoInv) + mul(&ret[1], &ret[1], &ret[0]) + mul(&ret[0], &ret[0], aa) + return e.sqrtAlignBLST(out, ret, ret, inp) +} + +func (e *fp2) sqrtAlignBLST(out, ret, sqrt, inp *fe2) bool { + + t0, t1 := new(fe2), new(fe2) + coeff := e.one() + e.square(t0, sqrt) + + // + fp2Sub(t1, t0, inp) + isSqrt := t1.isZero() + + // + fp2Add(t1, t0, inp) + flag := t1.isZero() + if flag { + coeff.set(sqrtMinus1) + } + isSqrt = flag || isSqrt + + // + sub(&t1[0], &t0[0], &inp[1]) + add(&t1[1], &t0[1], &inp[0]) + flag = t1.isZero() + if flag { + coeff.set(sqrtSqrtMinus1) + } + isSqrt = flag || isSqrt + + // + add(&t1[0], &t0[0], &inp[1]) + sub(&t1[1], &t0[1], &inp[0]) + flag = t1.isZero() + if flag { + + coeff.set(sqrtMinusSqrtMinus1) + } + isSqrt = flag || isSqrt + + e.mul(out, coeff, ret) + return isSqrt +} diff --git a/crypto/bls12381/fp2_arithmetic_x86.s b/crypto/bls12381/fp2_arithmetic_x86.s new file mode 100644 index 0000000000..d3d542150d --- /dev/null +++ b/crypto/bls12381/fp2_arithmetic_x86.s @@ -0,0 +1,4025 @@ +// +build amd64,!generic + +#include "textflag.h" +#include "funcdata.h" + +// assigned addition with modular reduction +// a = (a + b) % p +TEXT ·fp2AddAssign(SB), NOSPLIT, $0-16 + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, BP + MOVQ R13, BX + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, BP + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC BP, R12 + CMOVQCC BX, R13 + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + ADDQ 48(SI), R8 + ADCQ 56(SI), R9 + ADCQ 64(SI), R10 + ADCQ 72(SI), R11 + ADCQ 80(SI), R12 + ADCQ 88(SI), R13 + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, SI + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + MOVQ R8, 48(DI) + MOVQ R9, 56(DI) + MOVQ R10, 64(DI) + MOVQ R11, 72(DI) + MOVQ R12, 80(DI) + MOVQ R13, 88(DI) + + RET +/* | end */ + + +// addition with modular reduction +// c = (a + b) % p +TEXT ·fp2Add(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ c+0(FP), BP + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, SI + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + MOVQ R8, (BP) + MOVQ R9, 8(BP) + MOVQ R10, 16(BP) + MOVQ R11, 24(BP) + MOVQ R12, 32(BP) + MOVQ R13, 40(BP) + + MOVQ b+16(FP), SI + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + ADDQ 48(SI), R8 + ADCQ 56(SI), R9 + ADCQ 64(SI), R10 + ADCQ 72(SI), R11 + ADCQ 80(SI), R12 + ADCQ 88(SI), R13 + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, SI + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + MOVQ R8, 48(BP) + MOVQ R9, 56(BP) + MOVQ R10, 64(BP) + MOVQ R11, 72(BP) + MOVQ R12, 80(BP) + MOVQ R13, 88(BP) + + RET +/* | end */ + + +// addition without reduction check +// c = (a + b) +TEXT ·fp2Ladd(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ c+0(FP), AX + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + + MOVQ R8, (AX) + MOVQ R9, 8(AX) + MOVQ R10, 16(AX) + MOVQ R11, 24(AX) + MOVQ R12, 32(AX) + MOVQ R13, 40(AX) + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + ADDQ 48(SI), R8 + ADCQ 56(SI), R9 + ADCQ 64(SI), R10 + ADCQ 72(SI), R11 + ADCQ 80(SI), R12 + ADCQ 88(SI), R13 + + MOVQ R8, 48(AX) + MOVQ R9, 56(AX) + MOVQ R10, 64(AX) + MOVQ R11, 72(AX) + MOVQ R12, 80(AX) + MOVQ R13, 88(AX) + + RET +/* | end */ + + +// addition without reduction check +// c = (a + b) +TEXT ·fp2LaddAssign(SB), NOSPLIT, $0-16 + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + ADDQ 48(SI), R8 + ADCQ 56(SI), R9 + ADCQ 64(SI), R10 + ADCQ 72(SI), R11 + ADCQ 80(SI), R12 + ADCQ 88(SI), R13 + + MOVQ R8, 48(DI) + MOVQ R9, 56(DI) + MOVQ R10, 64(DI) + MOVQ R11, 72(DI) + MOVQ R12, 80(DI) + MOVQ R13, 88(DI) + + RET +/* | end */ + + +// subtraction with modular reduction +// c = (a - b) % p +TEXT ·fp2Sub(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + XORQ AX, AX + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, BP + MOVQ $0x1a0111ea397fe69a, BX + + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, BP + CMOVQCC AX, BX + + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ BP, R12 + ADCQ BX, R13 + + MOVQ c+0(FP), BP + MOVQ R8, (BP) + MOVQ R9, 8(BP) + MOVQ R10, 16(BP) + MOVQ R11, 24(BP) + MOVQ R12, 32(BP) + MOVQ R13, 40(BP) + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + SUBQ 48(SI), R8 + SBBQ 56(SI), R9 + SBBQ 64(SI), R10 + SBBQ 72(SI), R11 + SBBQ 80(SI), R12 + SBBQ 88(SI), R13 + + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, BP + MOVQ $0x1a0111ea397fe69a, BX + + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, BP + CMOVQCC AX, BX + + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ BP, R12 + ADCQ BX, R13 + + MOVQ c+0(FP), BP + MOVQ R8, 48(BP) + MOVQ R9, 56(BP) + MOVQ R10, 64(BP) + MOVQ R11, 72(BP) + MOVQ R12, 80(BP) + MOVQ R13, 88(BP) + + RET +/* | end */ + + +// assigned subtraction with modular reduction +// c = (a - b) % p +TEXT ·fp2SubAssign(SB), NOSPLIT, $0-16 + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + XORQ AX, AX + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, BP + MOVQ $0x1a0111ea397fe69a, BX + + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, BP + CMOVQCC AX, BX + + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ BP, R12 + ADCQ BX, R13 + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + SUBQ 48(SI), R8 + SBBQ 56(SI), R9 + SBBQ 64(SI), R10 + SBBQ 72(SI), R11 + SBBQ 80(SI), R12 + SBBQ 88(SI), R13 + + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, BP + MOVQ $0x1a0111ea397fe69a, BX + + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, BP + CMOVQCC AX, BX + + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ BP, R12 + ADCQ BX, R13 + + MOVQ R8, 48(DI) + MOVQ R9, 56(DI) + MOVQ R10, 64(DI) + MOVQ R11, 72(DI) + MOVQ R12, 80(DI) + MOVQ R13, 88(DI) + + RET +/* | end */ + + +// assigned doubling with modular reduction +// a = (a + a) % p +TEXT ·fp2DoubleAssign(SB), NOSPLIT, $0-8 + MOVQ a+0(FP), DI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, SI + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, SI + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + MOVQ R8, 48(DI) + MOVQ R9, 56(DI) + MOVQ R10, 64(DI) + MOVQ R11, 72(DI) + MOVQ R12, 80(DI) + MOVQ R13, 88(DI) + + RET +/* | end */ + + +// doubling with modular reduction +// c = (a + a) % p +TEXT ·fp2Double(SB), NOSPLIT, $0-16 + MOVQ a+8(FP), DI + MOVQ c+0(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, BP + MOVQ R13, BX + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, BP + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC BP, R12 + CMOVQCC BX, R13 + + MOVQ R8, (SI) + MOVQ R9, 8(SI) + MOVQ R10, 16(SI) + MOVQ R11, 24(SI) + MOVQ R12, 32(SI) + MOVQ R13, 40(SI) + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, BP + MOVQ R13, BX + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, BP + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC BP, R12 + CMOVQCC BX, R13 + + MOVQ R8, 48(SI) + MOVQ R9, 56(SI) + MOVQ R10, 64(SI) + MOVQ R11, 72(SI) + MOVQ R12, 80(SI) + MOVQ R13, 88(SI) + + RET +/* | end */ + + +// a0 = a0 - a1 +// a1 = a0 + a1 +TEXT ·mulByNonResidueAssign(SB), NOSPLIT, $0-8 + MOVQ a+0(FP), DI + XORQ AX, AX + + // a0 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // a0 - a1 + SUBQ 48(DI), R8 + SBBQ 56(DI), R9 + SBBQ 64(DI), R10 + SBBQ 72(DI), R11 + SBBQ 80(DI), R12 + SBBQ 88(DI), R13 + + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, BP + MOVQ $0x1a0111ea397fe69a, BX + + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, BP + CMOVQCC AX, BX + + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ BP, R12 + ADCQ BX, R13 + + // a0 + MOVQ (DI), R14 + MOVQ 8(DI), R15 + MOVQ 16(DI), CX + MOVQ 24(DI), DX + MOVQ 32(DI), BP + MOVQ 40(DI), BX + + // a0 = a0 - a1 + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + // a0 + a1 + ADDQ 48(DI), R14 + ADCQ 56(DI), R15 + ADCQ 64(DI), CX + ADCQ 72(DI), DX + ADCQ 80(DI), BP + ADCQ 88(DI), BX + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ CX, R10 + MOVQ DX, R11 + MOVQ BP, R12 + MOVQ BX, R13 + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R8 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R9 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, R10 + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, R11 + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, R12 + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, R13 + + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, CX + CMOVQCC R11, DX + CMOVQCC R12, BP + CMOVQCC R13, BX + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ CX, 64(DI) + MOVQ DX, 72(DI) + MOVQ BP, 80(DI) + MOVQ BX, 88(DI) + RET + + +// c0 = a0 - a1 +// c1 = a0 + a1 +TEXT ·mulByNonResidue(SB), NOSPLIT, $0-16 + MOVQ c+0(FP), SI + MOVQ a+8(FP), DI + XORQ AX, AX + + // a0 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // a0 - a1 + SUBQ 48(DI), R8 + SBBQ 56(DI), R9 + SBBQ 64(DI), R10 + SBBQ 72(DI), R11 + SBBQ 80(DI), R12 + SBBQ 88(DI), R13 + + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, BP + MOVQ $0x1a0111ea397fe69a, BX + + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, BP + CMOVQCC AX, BX + + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ BP, R12 + ADCQ BX, R13 + + MOVQ R8, (SI) + MOVQ R9, 8(SI) + MOVQ R10, 16(SI) + MOVQ R11, 24(SI) + MOVQ R12, 32(SI) + MOVQ R13, 40(SI) + + // a0 + MOVQ (DI), R14 + MOVQ 8(DI), R15 + MOVQ 16(DI), CX + MOVQ 24(DI), DX + MOVQ 32(DI), BP + MOVQ 40(DI), BX + + // a0 + a1 + ADDQ 48(DI), R14 + ADCQ 56(DI), R15 + ADCQ 64(DI), CX + ADCQ 72(DI), DX + ADCQ 80(DI), BP + ADCQ 88(DI), BX + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ CX, R10 + MOVQ DX, R11 + MOVQ BP, R12 + MOVQ BX, R13 + + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R8 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R9 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, R10 + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, R11 + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, R12 + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, R13 + + + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, CX + CMOVQCC R11, DX + CMOVQCC R12, BP + CMOVQCC R13, BX + + MOVQ R14, 48(SI) + MOVQ R15, 56(SI) + MOVQ CX, 64(SI) + MOVQ DX, 72(SI) + MOVQ BP, 80(SI) + MOVQ BX, 88(SI) + RET + + +TEXT ·wfp2Add(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ c+0(FP), DX + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), BP + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), BP + + MOVQ R8, (DX) + MOVQ R9, 8(DX) + MOVQ R10, 16(DX) + MOVQ R11, 24(DX) + MOVQ R12, 32(DX) + MOVQ R13, 40(DX) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ BP, R13 + + MOVQ $0xb9feffffffffaaab, DX + SUBQ DX, R8 + MOVQ $0x1eabfffeb153ffff, DX + SBBQ DX, R9 + MOVQ $0x6730d2a0f6b0f624, DX + SBBQ DX, R10 + MOVQ $0x64774b84f38512bf, DX + SBBQ DX, R11 + MOVQ $0x4b1ba7b6434bacd7, DX + SBBQ DX, R12 + MOVQ $0x1a0111ea397fe69a, DX + SBBQ DX, R13 + + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, BP + + MOVQ c+0(FP), DX + + MOVQ R14, 48(DX) + MOVQ R15, 56(DX) + MOVQ AX, 64(DX) + MOVQ BX, 72(DX) + MOVQ CX, 80(DX) + MOVQ BP, 88(DX) + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), BP + + ADDQ 96(SI), R8 + ADCQ 104(SI), R9 + ADCQ 112(SI), R10 + ADCQ 120(SI), R11 + ADCQ 128(SI), R12 + ADCQ 136(SI), R13 + ADCQ 144(SI), R14 + ADCQ 152(SI), R15 + ADCQ 160(SI), AX + ADCQ 168(SI), BX + ADCQ 176(SI), CX + ADCQ 184(SI), BP + + MOVQ R8, 96(DX) + MOVQ R9, 104(DX) + MOVQ R10, 112(DX) + MOVQ R11, 120(DX) + MOVQ R12, 128(DX) + MOVQ R13, 136(DX) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ BP, R13 + MOVQ $0xb9feffffffffaaab, DI + SUBQ DI, R8 + MOVQ $0x1eabfffeb153ffff, DI + SBBQ DI, R9 + MOVQ $0x6730d2a0f6b0f624, DI + SBBQ DI, R10 + MOVQ $0x64774b84f38512bf, DI + SBBQ DI, R11 + MOVQ $0x4b1ba7b6434bacd7, DI + SBBQ DI, R12 + MOVQ $0x1a0111ea397fe69a, DI + SBBQ DI, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, BP + + MOVQ R14, 144(DX) + MOVQ R15, 152(DX) + MOVQ AX, 160(DX) + MOVQ BX, 168(DX) + MOVQ CX, 176(DX) + MOVQ BP, 184(DX) + RET + + +TEXT ·wfp2AddAssign(SB), NOSPLIT, $0-16 + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), DX + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, BP + SUBQ BP, R8 + MOVQ $0x1eabfffeb153ffff, BP + SBBQ BP, R9 + MOVQ $0x6730d2a0f6b0f624, BP + SBBQ BP, R10 + MOVQ $0x64774b84f38512bf, BP + SBBQ BP, R11 + MOVQ $0x4b1ba7b6434bacd7, BP + SBBQ BP, R12 + MOVQ $0x1a0111ea397fe69a, BP + SBBQ BP, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), DX + + ADDQ 96(SI), R8 + ADCQ 104(SI), R9 + ADCQ 112(SI), R10 + ADCQ 120(SI), R11 + ADCQ 128(SI), R12 + ADCQ 136(SI), R13 + ADCQ 144(SI), R14 + ADCQ 152(SI), R15 + ADCQ 160(SI), AX + ADCQ 168(SI), BX + ADCQ 176(SI), CX + ADCQ 184(SI), DX + + MOVQ R8, 96(DI) + MOVQ R9, 104(DI) + MOVQ R10, 112(DI) + MOVQ R11, 120(DI) + MOVQ R12, 128(DI) + MOVQ R13, 136(DI) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, BP + SUBQ BP, R8 + MOVQ $0x1eabfffeb153ffff, BP + SBBQ BP, R9 + MOVQ $0x6730d2a0f6b0f624, BP + SBBQ BP, R10 + MOVQ $0x64774b84f38512bf, BP + SBBQ BP, R11 + MOVQ $0x4b1ba7b6434bacd7, BP + SBBQ BP, R12 + MOVQ $0x1a0111ea397fe69a, BP + SBBQ BP, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + + MOVQ R14, 144(DI) + MOVQ R15, 152(DI) + MOVQ AX, 160(DI) + MOVQ BX, 168(DI) + MOVQ CX, 176(DI) + MOVQ DX, 184(DI) + RET + + +TEXT ·wfp2AddMixed(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ c+0(FP), DX + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), BP + + ADDQ 96(SI), R8 + ADCQ 104(SI), R9 + ADCQ 112(SI), R10 + ADCQ 120(SI), R11 + ADCQ 128(SI), R12 + ADCQ 136(SI), R13 + ADCQ 144(SI), R14 + ADCQ 152(SI), R15 + ADCQ 160(SI), AX + ADCQ 168(SI), BX + ADCQ 176(SI), CX + ADCQ 184(SI), BP + + MOVQ R8, 96(DX) + MOVQ R9, 104(DX) + MOVQ R10, 112(DX) + MOVQ R11, 120(DX) + MOVQ R12, 128(DX) + MOVQ R13, 136(DX) + MOVQ R14, 144(DX) + MOVQ R15, 152(DX) + MOVQ AX, 160(DX) + MOVQ BX, 168(DX) + MOVQ CX, 176(DX) + MOVQ BP, 184(DX) + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), BP + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), BP + + MOVQ R8, (DX) + MOVQ R9, 8(DX) + MOVQ R10, 16(DX) + MOVQ R11, 24(DX) + MOVQ R12, 32(DX) + MOVQ R13, 40(DX) + + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ BP, R13 + MOVQ $0xb9feffffffffaaab, DI + SUBQ DI, R8 + MOVQ $0x1eabfffeb153ffff, DI + SBBQ DI, R9 + MOVQ $0x6730d2a0f6b0f624, DI + SBBQ DI, R10 + MOVQ $0x64774b84f38512bf, DI + SBBQ DI, R11 + MOVQ $0x4b1ba7b6434bacd7, DI + SBBQ DI, R12 + MOVQ $0x1a0111ea397fe69a, DI + SBBQ DI, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, BP + + MOVQ R14, 48(DX) + MOVQ R15, 56(DX) + MOVQ AX, 64(DX) + MOVQ BX, 72(DX) + MOVQ CX, 80(DX) + MOVQ BP, 88(DX) + RET + + +TEXT ·wfp2AddMixedAssign(SB), NOSPLIT, $0-16 + + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), BP + + ADDQ 96(SI), R8 + ADCQ 104(SI), R9 + ADCQ 112(SI), R10 + ADCQ 120(SI), R11 + ADCQ 128(SI), R12 + ADCQ 136(SI), R13 + ADCQ 144(SI), R14 + ADCQ 152(SI), R15 + ADCQ 160(SI), AX + ADCQ 168(SI), BX + ADCQ 176(SI), CX + ADCQ 184(SI), BP + + MOVQ R8, 96(DI) + MOVQ R9, 104(DI) + MOVQ R10, 112(DI) + MOVQ R11, 120(DI) + MOVQ R12, 128(DI) + MOVQ R13, 136(DI) + MOVQ R14, 144(DI) + MOVQ R15, 152(DI) + MOVQ AX, 160(DI) + MOVQ BX, 168(DI) + MOVQ CX, 176(DI) + MOVQ BP, 184(DI) + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), BP + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), BP + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ BP, R13 + MOVQ $0xb9feffffffffaaab, SI + SUBQ SI, R8 + MOVQ $0x1eabfffeb153ffff, SI + SBBQ SI, R9 + MOVQ $0x6730d2a0f6b0f624, SI + SBBQ SI, R10 + MOVQ $0x64774b84f38512bf, SI + SBBQ SI, R11 + MOVQ $0x4b1ba7b6434bacd7, SI + SBBQ SI, R12 + MOVQ $0x1a0111ea397fe69a, SI + SBBQ SI, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, BP + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ BP, 88(DI) + RET + + +TEXT ·wfp2Ladd(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ c+0(FP), DX + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), BP + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), BP + + MOVQ R8, (DX) + MOVQ R9, 8(DX) + MOVQ R10, 16(DX) + MOVQ R11, 24(DX) + MOVQ R12, 32(DX) + MOVQ R13, 40(DX) + MOVQ R14, 48(DX) + MOVQ R15, 56(DX) + MOVQ AX, 64(DX) + MOVQ BX, 72(DX) + MOVQ CX, 80(DX) + MOVQ BP, 88(DX) + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), BP + + ADDQ 96(SI), R8 + ADCQ 104(SI), R9 + ADCQ 112(SI), R10 + ADCQ 120(SI), R11 + ADCQ 128(SI), R12 + ADCQ 136(SI), R13 + ADCQ 144(SI), R14 + ADCQ 152(SI), R15 + ADCQ 160(SI), AX + ADCQ 168(SI), BX + ADCQ 176(SI), CX + ADCQ 184(SI), BP + + MOVQ R8, 96(DX) + MOVQ R9, 104(DX) + MOVQ R10, 112(DX) + MOVQ R11, 120(DX) + MOVQ R12, 128(DX) + MOVQ R13, 136(DX) + MOVQ R14, 144(DX) + MOVQ R15, 152(DX) + MOVQ AX, 160(DX) + MOVQ BX, 168(DX) + MOVQ CX, 176(DX) + MOVQ BP, 184(DX) + RET + + +TEXT ·wfp2LaddAssign(SB), NOSPLIT, $0-16 + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), BP + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), BP + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ BP, 88(DI) + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), BP + + ADDQ 96(SI), R8 + ADCQ 104(SI), R9 + ADCQ 112(SI), R10 + ADCQ 120(SI), R11 + ADCQ 128(SI), R12 + ADCQ 136(SI), R13 + ADCQ 144(SI), R14 + ADCQ 152(SI), R15 + ADCQ 160(SI), AX + ADCQ 168(SI), BX + ADCQ 176(SI), CX + ADCQ 184(SI), BP + + MOVQ R8, 96(DI) + MOVQ R9, 104(DI) + MOVQ R10, 112(DI) + MOVQ R11, 120(DI) + MOVQ R12, 128(DI) + MOVQ R13, 136(DI) + MOVQ R14, 144(DI) + MOVQ R15, 152(DI) + MOVQ AX, 160(DI) + MOVQ BX, 168(DI) + MOVQ CX, 176(DI) + MOVQ BP, 184(DI) + RET + + +TEXT ·wfp2Sub(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ c+0(FP), BP + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + SBBQ 48(SI), R14 + SBBQ 56(SI), R15 + SBBQ 64(SI), AX + SBBQ 72(SI), BX + SBBQ 80(SI), CX + SBBQ 88(SI), DX + + MOVQ R8, (BP) + MOVQ R9, 8(BP) + MOVQ R10, 16(BP) + MOVQ R11, 24(BP) + MOVQ R12, 32(BP) + MOVQ R13, 40(BP) + + MOVQ $0, SI + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC SI, R8 + CMOVQCC SI, R9 + CMOVQCC SI, R10 + CMOVQCC SI, R11 + CMOVQCC SI, R12 + CMOVQCC SI, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + MOVQ R14, 48(BP) + MOVQ R15, 56(BP) + MOVQ AX, 64(BP) + MOVQ BX, 72(BP) + MOVQ CX, 80(BP) + MOVQ DX, 88(BP) + + MOVQ b+16(FP), SI + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), DX + + SUBQ 96(SI), R8 + SBBQ 104(SI), R9 + SBBQ 112(SI), R10 + SBBQ 120(SI), R11 + SBBQ 128(SI), R12 + SBBQ 136(SI), R13 + SBBQ 144(SI), R14 + SBBQ 152(SI), R15 + SBBQ 160(SI), AX + SBBQ 168(SI), BX + SBBQ 176(SI), CX + SBBQ 184(SI), DX + + MOVQ R8, 96(BP) + MOVQ R9, 104(BP) + MOVQ R10, 112(BP) + MOVQ R11, 120(BP) + MOVQ R12, 128(BP) + MOVQ R13, 136(BP) + + MOVQ $0, SI + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC SI, R8 + CMOVQCC SI, R9 + CMOVQCC SI, R10 + CMOVQCC SI, R11 + CMOVQCC SI, R12 + CMOVQCC SI, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + MOVQ R14, 144(BP) + MOVQ R15, 152(BP) + MOVQ AX, 160(BP) + MOVQ BX, 168(BP) + MOVQ CX, 176(BP) + MOVQ DX, 184(BP) + RET + + +TEXT ·wfp2SubAssign(SB), NOSPLIT, $0-16 + + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + SBBQ 48(SI), R14 + SBBQ 56(SI), R15 + SBBQ 64(SI), AX + SBBQ 72(SI), BX + SBBQ 80(SI), CX + SBBQ 88(SI), DX + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ $0, BP + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC BP, R8 + CMOVQCC BP, R9 + CMOVQCC BP, R10 + CMOVQCC BP, R11 + CMOVQCC BP, R12 + CMOVQCC BP, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), DX + + SUBQ 96(SI), R8 + SBBQ 104(SI), R9 + SBBQ 112(SI), R10 + SBBQ 120(SI), R11 + SBBQ 128(SI), R12 + SBBQ 136(SI), R13 + SBBQ 144(SI), R14 + SBBQ 152(SI), R15 + SBBQ 160(SI), AX + SBBQ 168(SI), BX + SBBQ 176(SI), CX + SBBQ 184(SI), DX + + MOVQ R8, 96(DI) + MOVQ R9, 104(DI) + MOVQ R10, 112(DI) + MOVQ R11, 120(DI) + MOVQ R12, 128(DI) + MOVQ R13, 136(DI) + + MOVQ $0, BP + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC BP, R8 + CMOVQCC BP, R9 + CMOVQCC BP, R10 + CMOVQCC BP, R11 + CMOVQCC BP, R12 + CMOVQCC BP, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + MOVQ R14, 144(DI) + MOVQ R15, 152(DI) + MOVQ AX, 160(DI) + MOVQ BX, 168(DI) + MOVQ CX, 176(DI) + MOVQ DX, 184(DI) + RET + +TEXT ·wfp2SubMixed(SB), NOSPLIT, $0-24 + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + SBBQ 48(SI), R14 + SBBQ 56(SI), R15 + SBBQ 64(SI), AX + SBBQ 72(SI), BX + SBBQ 80(SI), CX + SBBQ 88(SI), DX + + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ $0, BP + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC BP, R8 + CMOVQCC BP, R9 + CMOVQCC BP, R10 + CMOVQCC BP, R11 + CMOVQCC BP, R12 + CMOVQCC BP, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + + MOVQ a+8(FP), DI + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), DX + + SUBQ 96(SI), R8 + SBBQ 104(SI), R9 + SBBQ 112(SI), R10 + SBBQ 120(SI), R11 + SBBQ 128(SI), R12 + SBBQ 136(SI), R13 + SBBQ 144(SI), R14 + SBBQ 152(SI), R15 + SBBQ 160(SI), AX + SBBQ 168(SI), BX + SBBQ 176(SI), CX + SBBQ 184(SI), DX + + MOVQ c+0(FP), DI + MOVQ R8, 96(DI) + MOVQ R9, 104(DI) + MOVQ R10, 112(DI) + MOVQ R11, 120(DI) + MOVQ R12, 128(DI) + MOVQ R13, 136(DI) + MOVQ R14, 144(DI) + MOVQ R15, 152(DI) + MOVQ AX, 160(DI) + MOVQ BX, 168(DI) + MOVQ CX, 176(DI) + MOVQ DX, 184(DI) + RET + +TEXT ·wfp2SubMixedAssign(SB), NOSPLIT, $0-16 + + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + SBBQ 48(SI), R14 + SBBQ 56(SI), R15 + SBBQ 64(SI), AX + SBBQ 72(SI), BX + SBBQ 80(SI), CX + SBBQ 88(SI), DX + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ $0, BP + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC BP, R8 + CMOVQCC BP, R9 + CMOVQCC BP, R10 + CMOVQCC BP, R11 + CMOVQCC BP, R12 + CMOVQCC BP, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), DX + + SUBQ 96(SI), R8 + SBBQ 104(SI), R9 + SBBQ 112(SI), R10 + SBBQ 120(SI), R11 + SBBQ 128(SI), R12 + SBBQ 136(SI), R13 + SBBQ 144(SI), R14 + SBBQ 152(SI), R15 + SBBQ 160(SI), AX + SBBQ 168(SI), BX + SBBQ 176(SI), CX + SBBQ 184(SI), DX + + MOVQ R8, 96(DI) + MOVQ R9, 104(DI) + MOVQ R10, 112(DI) + MOVQ R11, 120(DI) + MOVQ R12, 128(DI) + MOVQ R13, 136(DI) + MOVQ R14, 144(DI) + MOVQ R15, 152(DI) + MOVQ AX, 160(DI) + MOVQ BX, 168(DI) + MOVQ CX, 176(DI) + MOVQ DX, 184(DI) + RET + + +TEXT ·wfp2Double(SB), NOSPLIT, $0-16 + + MOVQ a+8(FP), DI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + ADCQ R14, R14 + ADCQ R15, R15 + ADCQ AX, AX + ADCQ BX, BX + ADCQ CX, CX + ADCQ DX, DX + + MOVQ c+0(FP), SI + MOVQ R8, (SI) + MOVQ R9, 8(SI) + MOVQ R10, 16(SI) + MOVQ R11, 24(SI) + MOVQ R12, 32(SI) + MOVQ R13, 40(SI) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, BP + SUBQ BP, R8 + MOVQ $0x1eabfffeb153ffff, BP + SBBQ BP, R9 + MOVQ $0x6730d2a0f6b0f624, BP + SBBQ BP, R10 + MOVQ $0x64774b84f38512bf, BP + SBBQ BP, R11 + MOVQ $0x4b1ba7b6434bacd7, BP + SBBQ BP, R12 + MOVQ $0x1a0111ea397fe69a, BP + SBBQ BP, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + + MOVQ R14, 48(SI) + MOVQ R15, 56(SI) + MOVQ AX, 64(SI) + MOVQ BX, 72(SI) + MOVQ CX, 80(SI) + MOVQ DX, 88(SI) + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), DX + + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + ADCQ R14, R14 + ADCQ R15, R15 + ADCQ AX, AX + ADCQ BX, BX + ADCQ CX, CX + ADCQ DX, DX + + MOVQ c+0(FP), SI + MOVQ R8, 96(SI) + MOVQ R9, 104(SI) + MOVQ R10, 112(SI) + MOVQ R11, 120(SI) + MOVQ R12, 128(SI) + MOVQ R13, 136(SI) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, DI + SUBQ DI, R8 + MOVQ $0x1eabfffeb153ffff, DI + SBBQ DI, R9 + MOVQ $0x6730d2a0f6b0f624, DI + SBBQ DI, R10 + MOVQ $0x64774b84f38512bf, DI + SBBQ DI, R11 + MOVQ $0x4b1ba7b6434bacd7, DI + SBBQ DI, R12 + MOVQ $0x1a0111ea397fe69a, DI + SBBQ DI, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + + MOVQ R14, 144(SI) + MOVQ R15, 152(SI) + MOVQ AX, 160(SI) + MOVQ BX, 168(SI) + MOVQ CX, 176(SI) + MOVQ DX, 184(SI) + RET + + TEXT ·wfp2DoubleAssign(SB), NOSPLIT, $0-8 + MOVQ a+0(FP), DI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + ADCQ R14, R14 + ADCQ R15, R15 + ADCQ AX, AX + ADCQ BX, BX + ADCQ CX, CX + ADCQ DX, DX + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, SI + SUBQ SI, R8 + MOVQ $0x1eabfffeb153ffff, SI + SBBQ SI, R9 + MOVQ $0x6730d2a0f6b0f624, SI + SBBQ SI, R10 + MOVQ $0x64774b84f38512bf, SI + SBBQ SI, R11 + MOVQ $0x4b1ba7b6434bacd7, SI + SBBQ SI, R12 + MOVQ $0x1a0111ea397fe69a, SI + SBBQ SI, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + + MOVQ 96(DI), R8 + MOVQ 104(DI), R9 + MOVQ 112(DI), R10 + MOVQ 120(DI), R11 + MOVQ 128(DI), R12 + MOVQ 136(DI), R13 + MOVQ 144(DI), R14 + MOVQ 152(DI), R15 + MOVQ 160(DI), AX + MOVQ 168(DI), BX + MOVQ 176(DI), CX + MOVQ 184(DI), DX + + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + ADCQ R14, R14 + ADCQ R15, R15 + ADCQ AX, AX + ADCQ BX, BX + ADCQ CX, CX + ADCQ DX, DX + + MOVQ R8, 96(DI) + MOVQ R9, 104(DI) + MOVQ R10, 112(DI) + MOVQ R11, 120(DI) + MOVQ R12, 128(DI) + MOVQ R13, 136(DI) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, SI + SUBQ SI, R8 + MOVQ $0x1eabfffeb153ffff, SI + SBBQ SI, R9 + MOVQ $0x6730d2a0f6b0f624, SI + SBBQ SI, R10 + MOVQ $0x64774b84f38512bf, SI + SBBQ SI, R11 + MOVQ $0x4b1ba7b6434bacd7, SI + SBBQ SI, R12 + MOVQ $0x1a0111ea397fe69a, SI + SBBQ SI, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + + MOVQ R14, 144(DI) + MOVQ R15, 152(DI) + MOVQ AX, 160(DI) + MOVQ BX, 168(DI) + MOVQ CX, 176(DI) + MOVQ DX, 184(DI) + RET + + + +// c1 = a0 + a1 +// c0 = a0 - a1 +TEXT ·wfp2MulByNonResidue(SB), NOSPLIT, $0-16 + MOVQ c+0(FP), SI + MOVQ a+8(FP), DI + + // a0 - a1 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ 96(DI), R8 + SBBQ 104(DI), R9 + SBBQ 112(DI), R10 + SBBQ 120(DI), R11 + SBBQ 128(DI), R12 + SBBQ 136(DI), R13 + SBBQ 144(DI), R14 + SBBQ 152(DI), R15 + SBBQ 160(DI), AX + SBBQ 168(DI), BX + SBBQ 176(DI), CX + SBBQ 184(DI), DX + + MOVQ R8, (SI) + MOVQ R9, 8(SI) + MOVQ R10, 16(SI) + MOVQ R11, 24(SI) + MOVQ R12, 32(SI) + MOVQ R13, 40(SI) + + MOVQ $0, BP + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC BP, R8 + CMOVQCC BP, R9 + CMOVQCC BP, R10 + CMOVQCC BP, R11 + CMOVQCC BP, R12 + CMOVQCC BP, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + MOVQ R14, 48(SI) + MOVQ R15, 56(SI) + MOVQ AX, 64(SI) + MOVQ BX, 72(SI) + MOVQ CX, 80(SI) + MOVQ DX, 88(SI) + + // a0 + a1 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + ADDQ 96(DI), R8 + ADCQ 104(DI), R9 + ADCQ 112(DI), R10 + ADCQ 120(DI), R11 + ADCQ 128(DI), R12 + ADCQ 136(DI), R13 + ADCQ 144(DI), R14 + ADCQ 152(DI), R15 + ADCQ 160(DI), AX + ADCQ 168(DI), BX + ADCQ 176(DI), CX + ADCQ 184(DI), DX + + MOVQ R8, 96(SI) + MOVQ R9, 104(SI) + MOVQ R10, 112(SI) + MOVQ R11, 120(SI) + MOVQ R12, 128(SI) + MOVQ R13, 136(SI) + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, BP + SUBQ BP, R8 + MOVQ $0x1eabfffeb153ffff, BP + SBBQ BP, R9 + MOVQ $0x6730d2a0f6b0f624, BP + SBBQ BP, R10 + MOVQ $0x64774b84f38512bf, BP + SBBQ BP, R11 + MOVQ $0x4b1ba7b6434bacd7, BP + SBBQ BP, R12 + MOVQ $0x1a0111ea397fe69a, BP + SBBQ BP, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + MOVQ R14, 144(SI) + MOVQ R15, 152(SI) + MOVQ AX, 160(SI) + MOVQ BX, 168(SI) + MOVQ CX, 176(SI) + MOVQ DX, 184(SI) + + RET + + +TEXT ·wfp2MulByNonResidueAssign(SB), NOSPLIT, $64-8 + MOVQ a+0(FP), DI + + // a0 - a1 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ 96(DI), R8 + SBBQ 104(DI), R9 + SBBQ 112(DI), R10 + SBBQ 120(DI), R11 + SBBQ 128(DI), R12 + SBBQ 136(DI), R13 + SBBQ 144(DI), R14 + SBBQ 152(DI), R15 + SBBQ 160(DI), AX + SBBQ 168(DI), BX + SBBQ 176(DI), CX + SBBQ 184(DI), DX + + MOVQ R8, (SP) + MOVQ R9, 8(SP) + MOVQ R10, 16(SP) + MOVQ R11, 24(SP) + MOVQ R12, 32(SP) + MOVQ R13, 40(SP) + + MOVQ $0, BP + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC BP, R8 + CMOVQCC BP, R9 + CMOVQCC BP, R10 + CMOVQCC BP, R11 + CMOVQCC BP, R12 + CMOVQCC BP, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + + MOVQ 48(DI), R8 + MOVQ 56(DI), R9 + MOVQ 64(DI), R10 + MOVQ 72(DI), R11 + MOVQ 80(DI), R12 + MOVQ 88(DI), R13 + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + + // a0 + a1 + MOVQ (DI), R14 + MOVQ 8(DI), R15 + MOVQ 16(DI), AX + MOVQ 24(DI), BX + MOVQ 32(DI), CX + MOVQ 40(DI), DX + + + ADDQ 96(DI), R14 + ADCQ 104(DI), R15 + ADCQ 112(DI), AX + ADCQ 120(DI), BX + ADCQ 128(DI), CX + ADCQ 136(DI), DX + ADCQ 144(DI), R8 + ADCQ 152(DI), R9 + ADCQ 160(DI), R10 + ADCQ 168(DI), R11 + ADCQ 176(DI), R12 + ADCQ 184(DI), R13 + + MOVQ R14, 96(DI) + MOVQ R15, 104(DI) + MOVQ AX, 112(DI) + MOVQ BX, 120(DI) + MOVQ CX, 128(DI) + MOVQ DX, 136(DI) + + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, AX + MOVQ R11, BX + MOVQ R12, CX + MOVQ R13, DX + MOVQ $0xb9feffffffffaaab, BP + SUBQ BP, R14 + MOVQ $0x1eabfffeb153ffff, BP + SBBQ BP, R15 + MOVQ $0x6730d2a0f6b0f624, BP + SBBQ BP, AX + MOVQ $0x64774b84f38512bf, BP + SBBQ BP, BX + MOVQ $0x4b1ba7b6434bacd7, BP + SBBQ BP, CX + MOVQ $0x1a0111ea397fe69a, BP + SBBQ BP, DX + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC AX, R10 + CMOVQCC BX, R11 + CMOVQCC CX, R12 + CMOVQCC DX, R13 + MOVQ R8, 144(DI) + MOVQ R9, 152(DI) + MOVQ R10, 160(DI) + MOVQ R11, 168(DI) + MOVQ R12, 176(DI) + MOVQ R13, 184(DI) + + MOVQ (SP), R8 + MOVQ 8(SP), R9 + MOVQ 16(SP), R10 + MOVQ 24(SP), R11 + MOVQ 32(SP), R12 + MOVQ 40(SP), R13 + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + RET + + +TEXT ·wfp2SquareADX(SB), NOSPLIT, $96-16 + MOVQ a+8(FP), DI + + // a0 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // a1 + MOVQ 48(DI), R15 + MOVQ 56(DI), BX + MOVQ 64(DI), CX + MOVQ 72(DI), DX + MOVQ 80(DI), SI + MOVQ 88(DI), R14 + + // a0 + a1 + ADDQ R8, R15 + ADCQ R9, BX + ADCQ R10, CX + ADCQ R11, DX + ADCQ R12, SI + ADCQ R13, R14 + + XORQ AX, AX + + // store a0 + a1 + MOVQ R15, (SP) + MOVQ BX, 8(SP) + MOVQ CX, 16(SP) + MOVQ DX, 24(SP) + MOVQ SI, 32(SP) + MOVQ R14, 40(SP) + + // a0 - a1 + SUBQ 48(DI), R8 + SBBQ 56(DI), R9 + SBBQ 64(DI), R10 + SBBQ 72(DI), R11 + SBBQ 80(DI), R12 + SBBQ 88(DI), R13 + + + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, SI + MOVQ $0x1a0111ea397fe69a, BX + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, SI + CMOVQCC AX, BX + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ SI, R12 + ADCQ BX, R13 + + // a0 - a1 + MOVQ R8, 48(SP) + MOVQ R9, 56(SP) + MOVQ R10, 64(SP) + MOVQ R11, 72(SP) + MOVQ R12, 80(SP) + MOVQ R13, 88(SP) + + // c0 = (a0 + a1)(a0 - a1) + MOVQ c+0(FP), SI + +/* i0 */ + + XORQ BX, BX + MOVQ 48(SP), DX + + // | a0 * b0 + MULXQ (SP), AX, CX + MOVQ AX, (SI) + + // | a0 * b1 + MULXQ 8(SP), AX, BP + ADCXQ AX, CX + + // | a0 * b2 + MULXQ 16(SP), AX, R9 + ADCXQ AX, BP + + // | a0 * b3 + MULXQ 24(SP), AX, R10 + ADCXQ AX, R9 + + // | a0 * b4 + MULXQ 32(SP), AX, R11 + ADCXQ AX, R10 + + // | a0 * b5 + MULXQ 40(SP), AX, R12 + ADCXQ AX, R11 + ADCXQ BX, R12 + +/* i1 */ + + MOVQ 56(SP), DX + + // | a1 * b0 + MULXQ (SP), AX, R13 + ADOXQ AX, CX + ADCXQ R13, BP + MOVQ CX, 8(SI) + + // | a1 * b1 + MULXQ 8(SP), AX, R13 + ADOXQ AX, BP + ADCXQ R13, R9 + + // | a1 * b2 + MULXQ 16(SP), AX, R13 + ADOXQ AX, R9 + ADCXQ R13, R10 + + // | a1 * b3 + MULXQ 24(SP), AX, R13 + ADOXQ AX, R10 + ADCXQ R13, R11 + + // | a1 * b4 + MULXQ 32(SP), AX, R13 + ADOXQ AX, R11 + ADCXQ R13, R12 + + // | a1 * b5 + MULXQ 40(SP), AX, R13 + ADOXQ AX, R12 + ADOXQ BX, R13 + ADCXQ BX, R13 + +/* i2 */ + + MOVQ 64(SP), DX + + // | a2 * b0 + MULXQ (SP), AX, R14 + ADOXQ AX, BP + ADCXQ R14, R9 + MOVQ BP, 16(SI) + + // | a2 * b1 + MULXQ 8(SP), AX, R14 + ADOXQ AX, R9 + ADCXQ R14, R10 + + // | a2 * b2 + MULXQ 16(SP), AX, R14 + ADOXQ AX, R10 + ADCXQ R14, R11 + + // | a2 * b3 + MULXQ 24(SP), AX, R14 + ADOXQ AX, R11 + ADCXQ R14, R12 + + // | a2 * b4 + MULXQ 32(SP), AX, R14 + ADOXQ AX, R12 + ADCXQ R14, R13 + + // | a2 * b5 + MULXQ 40(SP), AX, R14 + ADOXQ AX, R13 + ADOXQ BX, R14 + ADCXQ BX, R14 + +/* i3 */ + + MOVQ 72(SP), DX + + // | a3 * b0 + MULXQ (SP), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + MOVQ R9, 24(SI) + + // | a3 * b1 + MULXQ 8(SP), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | a3 * b2 + MULXQ 16(SP), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | a3 * b3 + MULXQ 24(SP), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | a3 * b4 + MULXQ 32(SP), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | a3 * b5 + MULXQ 40(SP), AX, R15 + ADOXQ AX, R14 + ADOXQ BX, R15 + ADCXQ BX, R15 + +/* i4 */ + + MOVQ 80(SP), DX + + // | a4 * b0 + MULXQ (SP), AX, CX + ADOXQ AX, R10 + ADCXQ CX, R11 + MOVQ R10, 32(SI) + + // | a4 * b1 + MULXQ 8(SP), AX, CX + ADOXQ AX, R11 + ADCXQ CX, R12 + + // | a4 * b2 + MULXQ 16(SP), AX, CX + ADOXQ AX, R12 + ADCXQ CX, R13 + + // | a4 * b3 + MULXQ 24(SP), AX, CX + ADOXQ AX, R13 + ADCXQ CX, R14 + + // | a4 * b4 + MULXQ 32(SP), AX, CX + ADOXQ AX, R14 + ADCXQ CX, R15 + + // | a4 * b5 + MULXQ 40(SP), AX, CX + ADOXQ AX, R15 + ADOXQ BX, CX + ADCXQ BX, CX + +/* i5 */ + + MOVQ 88(SP), DX + + // | a5 * b0 + MULXQ (SP), AX, R8 + ADOXQ AX, R11 + ADCXQ R8, R12 + MOVQ R11, 40(SI) + + // | a5 * b1 + MULXQ 8(SP), AX, R8 + ADOXQ AX, R12 + ADCXQ R8, R13 + + // | a5 * b2 + MULXQ 16(SP), AX, R8 + ADOXQ AX, R13 + ADCXQ R8, R14 + + // | a5 * b3 + MULXQ 24(SP), AX, R8 + ADOXQ AX, R14 + ADCXQ R8, R15 + + // | a5 * b4 + MULXQ 32(SP), AX, R8 + ADOXQ AX, R15 + ADCXQ R8, CX + + // | a5 * b5 + MULXQ 40(SP), AX, R8 + ADOXQ AX, CX + ADOXQ BX, R8 + ADCXQ BX, R8 + + + // w0 stored + MOVQ R12, 48(SI) + MOVQ R13, 56(SI) + MOVQ R14, 64(SI) + MOVQ R15, 72(SI) + MOVQ CX, 80(SI) + MOVQ R8, 88(SI) + + + // a0 + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // 2a0 + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + + MOVQ R8, (SP) + MOVQ R9, 8(SP) + MOVQ R10, 16(SP) + MOVQ R11, 24(SP) + MOVQ R12, 32(SP) + MOVQ R13, 40(SP) + + XORQ BX, BX + +/* i0 */ + + MOVQ 48(DI), DX + + // | a0 * b0 + MULXQ (SP), AX, CX + MOVQ AX, 96(SI) + + // | a0 * b1 + MULXQ 8(SP), AX, BP + ADCXQ AX, CX + + // | a0 * b2 + MULXQ 16(SP), AX, R9 + ADCXQ AX, BP + + // | a0 * b3 + MULXQ 24(SP), AX, R10 + ADCXQ AX, R9 + + // | a0 * b4 + MULXQ 32(SP), AX, R11 + ADCXQ AX, R10 + + // | a0 * b5 + MULXQ 40(SP), AX, R12 + ADCXQ AX, R11 + ADCXQ BX, R12 + + // | + +/* i1 */ + + MOVQ 56(DI), DX + + // | a1 * b0 + MULXQ (SP), AX, R13 + ADOXQ AX, CX + ADCXQ R13, BP + MOVQ CX, 104(SI) + + // | a1 * b1 + MULXQ 8(SP), AX, R13 + ADOXQ AX, BP + ADCXQ R13, R9 + + // | a1 * b2 + MULXQ 16(SP), AX, R13 + ADOXQ AX, R9 + ADCXQ R13, R10 + + // | a1 * b3 + MULXQ 24(SP), AX, R13 + ADOXQ AX, R10 + ADCXQ R13, R11 + + // | a1 * b4 + MULXQ 32(SP), AX, R13 + ADOXQ AX, R11 + ADCXQ R13, R12 + + // | a1 * b5 + MULXQ 40(SP), AX, R13 + ADOXQ AX, R12 + ADOXQ BX, R13 + ADCXQ BX, R13 + +/* i2 */ + + MOVQ 64(DI), DX + + // | a2 * b0 + MULXQ (SP), AX, R14 + ADOXQ AX, BP + ADCXQ R14, R9 + MOVQ BP, 112(SI) + + // | a2 * b1 + MULXQ 8(SP), AX, R14 + ADOXQ AX, R9 + ADCXQ R14, R10 + + // | a2 * b2 + MULXQ 16(SP), AX, R14 + ADOXQ AX, R10 + ADCXQ R14, R11 + + // | a2 * b3 + MULXQ 24(SP), AX, R14 + ADOXQ AX, R11 + ADCXQ R14, R12 + + // | a2 * b4 + MULXQ 32(SP), AX, R14 + ADOXQ AX, R12 + ADCXQ R14, R13 + + // | a2 * b5 + MULXQ 40(SP), AX, R14 + ADOXQ AX, R13 + ADOXQ BX, R14 + ADCXQ BX, R14 + +/* i3 */ + + MOVQ 72(DI), DX + + // | a3 * b0 + MULXQ (SP), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + MOVQ R9, 120(SI) + + // | a3 * b1 + MULXQ 8(SP), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | a3 * b2 + MULXQ 16(SP), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | a3 * b3 + MULXQ 24(SP), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | a3 * b4 + MULXQ 32(SP), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | a3 * b5 + MULXQ 40(SP), AX, R15 + ADOXQ AX, R14 + ADOXQ BX, R15 + ADCXQ BX, R15 + +/* i4 */ + + MOVQ 80(DI), DX + + // | a4 * b0 + MULXQ (SP), AX, CX + ADOXQ AX, R10 + ADCXQ CX, R11 + MOVQ R10, 128(SI) + + // | a4 * b1 + MULXQ 8(SP), AX, CX + ADOXQ AX, R11 + ADCXQ CX, R12 + + // | a4 * b2 + MULXQ 16(SP), AX, CX + ADOXQ AX, R12 + ADCXQ CX, R13 + + // | a4 * b3 + MULXQ 24(SP), AX, CX + ADOXQ AX, R13 + ADCXQ CX, R14 + + // | a4 * b4 + MULXQ 32(SP), AX, CX + ADOXQ AX, R14 + ADCXQ CX, R15 + + // | a4 * b5 + MULXQ 40(SP), AX, CX + ADOXQ AX, R15 + ADOXQ BX, CX + ADCXQ BX, CX + +/* i5 */ + + MOVQ 88(DI), DX + + // | a5 * b0 + MULXQ (SP), AX, DI + ADOXQ AX, R11 + ADCXQ DI, R12 + MOVQ R11, 136(SI) + + // | a5 * b1 + MULXQ 8(SP), AX, DI + ADOXQ AX, R12 + ADCXQ DI, R13 + + // | a5 * b2 + MULXQ 16(SP), AX, DI + ADOXQ AX, R13 + ADCXQ DI, R14 + + // | a5 * b3 + MULXQ 24(SP), AX, DI + ADOXQ AX, R14 + ADCXQ DI, R15 + + // | a5 * b4 + MULXQ 32(SP), AX, DI + ADOXQ AX, R15 + ADCXQ DI, CX + + // | a5 * b5 + MULXQ 40(SP), AX, DI + ADOXQ AX, CX + ADOXQ BX, DI + ADCXQ BX, DI + + MOVQ R12, 144(SI) + MOVQ R13, 152(SI) + MOVQ R14, 160(SI) + MOVQ R15, 168(SI) + MOVQ CX, 176(SI) + MOVQ DI, 184(SI) + RET + + +TEXT ·wfp2MulADX(SB), NOSPLIT, $192-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + +// a0b0 +/* i0 */ + XORQ BX, BX + MOVQ (SI), DX + + // | a0 * b0 + MULXQ (DI), AX, CX + MOVQ AX, (SP) + + // | a0 * b1 + MULXQ 8(DI), AX, BP + ADCXQ AX, CX + + // | a0 * b2 + MULXQ 16(DI), AX, R9 + ADCXQ AX, BP + + // | a0 * b3 + MULXQ 24(DI), AX, R10 + ADCXQ AX, R9 + + // | a0 * b4 + MULXQ 32(DI), AX, R11 + ADCXQ AX, R10 + + // | a0 * b5 + MULXQ 40(DI), AX, R12 + ADCXQ AX, R11 + ADCXQ BX, R12 + +/* i1 */ + + MOVQ 8(SI), DX + + // | a1 * b0 + MULXQ (DI), AX, R13 + ADOXQ AX, CX + ADCXQ R13, BP + MOVQ CX, 8(SP) + + // | a1 * b1 + MULXQ 8(DI), AX, R13 + ADOXQ AX, BP + ADCXQ R13, R9 + + // | a1 * b2 + MULXQ 16(DI), AX, R13 + ADOXQ AX, R9 + ADCXQ R13, R10 + + // | a1 * b3 + MULXQ 24(DI), AX, R13 + ADOXQ AX, R10 + ADCXQ R13, R11 + + // | a1 * b4 + MULXQ 32(DI), AX, R13 + ADOXQ AX, R11 + ADCXQ R13, R12 + + // | a1 * b5 + MULXQ 40(DI), AX, R13 + ADOXQ AX, R12 + ADOXQ BX, R13 + ADCXQ BX, R13 + +/* i2 */ + + MOVQ 16(SI), DX + + // | a2 * b0 + MULXQ (DI), AX, R14 + ADOXQ AX, BP + ADCXQ R14, R9 + MOVQ BP, 16(SP) + + // | a2 * b1 + MULXQ 8(DI), AX, R14 + ADOXQ AX, R9 + ADCXQ R14, R10 + + // | a2 * b2 + MULXQ 16(DI), AX, R14 + ADOXQ AX, R10 + ADCXQ R14, R11 + + // | a2 * b3 + MULXQ 24(DI), AX, R14 + ADOXQ AX, R11 + ADCXQ R14, R12 + + // | a2 * b4 + MULXQ 32(DI), AX, R14 + ADOXQ AX, R12 + ADCXQ R14, R13 + + // | a2 * b5 + MULXQ 40(DI), AX, R14 + ADOXQ AX, R13 + ADOXQ BX, R14 + ADCXQ BX, R14 + +/* i3 */ + + MOVQ 24(SI), DX + + // | a3 * b0 + MULXQ (DI), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + MOVQ R9, 24(SP) + + // | a3 * b1 + MULXQ 8(DI), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | a3 * b2 + MULXQ 16(DI), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | a3 * b3 + MULXQ 24(DI), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | a3 * b4 + MULXQ 32(DI), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | a3 * b5 + MULXQ 40(DI), AX, R15 + ADOXQ AX, R14 + ADOXQ BX, R15 + ADCXQ BX, R15 + +/* i4 */ + + MOVQ 32(SI), DX + + // | a4 * b0 + MULXQ (DI), AX, CX + ADOXQ AX, R10 + ADCXQ CX, R11 + MOVQ R10, 32(SP) + + // | a4 * b1 + MULXQ 8(DI), AX, CX + ADOXQ AX, R11 + ADCXQ CX, R12 + + // | a4 * b2 + MULXQ 16(DI), AX, CX + ADOXQ AX, R12 + ADCXQ CX, R13 + + // | a4 * b3 + MULXQ 24(DI), AX, CX + ADOXQ AX, R13 + ADCXQ CX, R14 + + // | a4 * b4 + MULXQ 32(DI), AX, CX + ADOXQ AX, R14 + ADCXQ CX, R15 + + // | a4 * b5 + MULXQ 40(DI), AX, CX + ADOXQ AX, R15 + ADOXQ BX, CX + ADCXQ BX, CX + +/* i5 */ + + + MOVQ 40(SI), DX + + // | a5 * b0 + MULXQ (DI), AX, R8 + ADOXQ AX, R11 + ADCXQ R8, R12 + MOVQ R11, 40(SP) + + // | a5 * b1 + MULXQ 8(DI), AX, R8 + ADOXQ AX, R12 + ADCXQ R8, R13 + + // | a5 * b2 + MULXQ 16(DI), AX, R8 + ADOXQ AX, R13 + ADCXQ R8, R14 + + // | a5 * b3 + MULXQ 24(DI), AX, R8 + ADOXQ AX, R14 + ADCXQ R8, R15 + + // | a5 * b4 + MULXQ 32(DI), AX, R8 + ADOXQ AX, R15 + ADCXQ R8, CX + + // | a5 * b5 + MULXQ 40(DI), AX, R8 + ADOXQ AX, CX + ADOXQ BX, R8 + ADCXQ BX, R8 + + + // a0b0 stored (0, 88)SP + MOVQ R12, 48(SP) + MOVQ R13, 56(SP) + MOVQ R14, 64(SP) + MOVQ R15, 72(SP) + MOVQ CX, 80(SP) + MOVQ R8, 88(SP) + + +// a1b1 + +/* i0 */ + XORQ BX, BX + MOVQ 48(SI), DX + + // | a0 * b0 + MULXQ 48(DI), AX, CX + MOVQ AX, 96(SP) + + // | a0 * b1 + MULXQ 56(DI), AX, BP + ADCXQ AX, CX + + // | a0 * b2 + MULXQ 64(DI), AX, R9 + ADCXQ AX, BP + + // | a0 * b3 + MULXQ 72(DI), AX, R10 + ADCXQ AX, R9 + + // | a0 * b4 + MULXQ 80(DI), AX, R11 + ADCXQ AX, R10 + + // | a0 * b5 + MULXQ 88(DI), AX, R12 + ADCXQ AX, R11 + ADCXQ BX, R12 + +/* i1 */ + + MOVQ 56(SI), DX + + // | a1 * b0 + MULXQ 48(DI), AX, R13 + ADOXQ AX, CX + ADCXQ R13, BP + MOVQ CX, 104(SP) + + // | a1 * b1 + MULXQ 56(DI), AX, R13 + ADOXQ AX, BP + ADCXQ R13, R9 + + // | a1 * b2 + MULXQ 64(DI), AX, R13 + ADOXQ AX, R9 + ADCXQ R13, R10 + + // | a1 * b3 + MULXQ 72(DI), AX, R13 + ADOXQ AX, R10 + ADCXQ R13, R11 + + // | a1 * b4 + MULXQ 80(DI), AX, R13 + ADOXQ AX, R11 + ADCXQ R13, R12 + + // | a1 * b5 + MULXQ 88(DI), AX, R13 + ADOXQ AX, R12 + ADOXQ BX, R13 + ADCXQ BX, R13 + +/* i2 */ + + MOVQ 64(SI), DX + + // | a2 * b0 + MULXQ 48(DI), AX, R14 + ADOXQ AX, BP + ADCXQ R14, R9 + MOVQ BP, 112(SP) + + // | a2 * b1 + MULXQ 56(DI), AX, R14 + ADOXQ AX, R9 + ADCXQ R14, R10 + + // | a2 * b2 + MULXQ 64(DI), AX, R14 + ADOXQ AX, R10 + ADCXQ R14, R11 + + // | a2 * b3 + MULXQ 72(DI), AX, R14 + ADOXQ AX, R11 + ADCXQ R14, R12 + + // | a2 * b4 + MULXQ 80(DI), AX, R14 + ADOXQ AX, R12 + ADCXQ R14, R13 + + // | a2 * b5 + MULXQ 88(DI), AX, R14 + ADOXQ AX, R13 + ADOXQ BX, R14 + ADCXQ BX, R14 + +/* i3 */ + + MOVQ 72(SI), DX + + // | a3 * b0 + MULXQ 48(DI), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + MOVQ R9, 120(SP) + + // | a3 * b1 + MULXQ 56(DI), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | a3 * b2 + MULXQ 64(DI), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | a3 * b3 + MULXQ 72(DI), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | a3 * b4 + MULXQ 80(DI), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | a3 * b5 + MULXQ 88(DI), AX, R15 + ADOXQ AX, R14 + ADOXQ BX, R15 + ADCXQ BX, R15 + +/* i4 */ + + MOVQ 80(SI), DX + + // | a4 * b0 + MULXQ 48(DI), AX, CX + ADOXQ AX, R10 + ADCXQ CX, R11 + MOVQ R10, 128(SP) + + // | a4 * b1 + MULXQ 56(DI), AX, CX + ADOXQ AX, R11 + ADCXQ CX, R12 + + // | a4 * b2 + MULXQ 64(DI), AX, CX + ADOXQ AX, R12 + ADCXQ CX, R13 + + // | a4 * b3 + MULXQ 72(DI), AX, CX + ADOXQ AX, R13 + ADCXQ CX, R14 + + // | a4 * b4 + MULXQ 80(DI), AX, CX + ADOXQ AX, R14 + ADCXQ CX, R15 + + // | a4 * b5 + MULXQ 88(DI), AX, CX + ADOXQ AX, R15 + ADOXQ BX, CX + ADCXQ BX, CX + +/* i5 */ + + MOVQ 88(SI), DX + + // | a5 * b0 + MULXQ 48(DI), AX, R8 + ADOXQ AX, R11 + ADCXQ R8, R12 + MOVQ R11, 136(SP) + + // | a5 * b1 + MULXQ 56(DI), AX, R8 + ADOXQ AX, R12 + ADCXQ R8, R13 + + // | a5 * b2 + MULXQ 64(DI), AX, R8 + ADOXQ AX, R13 + ADCXQ R8, R14 + + // | a5 * b3 + MULXQ 72(DI), AX, R8 + ADOXQ AX, R14 + ADCXQ R8, R15 + + // | a5 * b4 + MULXQ 80(DI), AX, R8 + ADOXQ AX, R15 + ADCXQ R8, CX + + // | a5 * b5 + MULXQ 88(DI), AX, R8 + ADOXQ AX, CX + ADOXQ BX, R8 + ADCXQ BX, R8 + + +// a1b1 stored (96, 184)SP + MOVQ R12, 144(SP) + MOVQ R13, 152(SP) + MOVQ R14, 160(SP) + MOVQ R15, 168(SP) + MOVQ CX, 176(SP) + MOVQ R8, 184(SP) + + +// a0b0 - a1b1 + MOVQ (SP), R8 + MOVQ 8(SP), R9 + MOVQ 16(SP), R10 + MOVQ 24(SP), R11 + MOVQ 32(SP), R12 + MOVQ 40(SP), R13 + MOVQ 48(SP), R14 + MOVQ 56(SP), R15 + MOVQ 64(SP), AX + MOVQ 72(SP), BX + MOVQ 80(SP), CX + MOVQ 88(SP), DX + SUBQ 96(SP), R8 + SBBQ 104(SP), R9 + SBBQ 112(SP), R10 + SBBQ 120(SP), R11 + SBBQ 128(SP), R12 + SBBQ 136(SP), R13 + // todo this can be avoided + SBBQ 144(SP), R14 + SBBQ 152(SP), R15 + SBBQ 160(SP), AX + SBBQ 168(SP), BX + SBBQ 176(SP), CX + SBBQ 184(SP), DX + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + MOVQ $0, SI + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC SI, R8 + CMOVQCC SI, R9 + CMOVQCC SI, R10 + CMOVQCC SI, R11 + CMOVQCC SI, R12 + CMOVQCC SI, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + +// a0b0 + a1b1 + MOVQ (SP), R8 + MOVQ 8(SP), R9 + MOVQ 16(SP), R10 + MOVQ 24(SP), R11 + MOVQ 32(SP), R12 + MOVQ 40(SP), R13 + MOVQ 48(SP), R14 + MOVQ 56(SP), R15 + MOVQ 64(SP), AX + MOVQ 72(SP), BX + MOVQ 80(SP), CX + MOVQ 88(SP), DX + ADDQ 96(SP), R8 + ADCQ 104(SP), R9 + ADCQ 112(SP), R10 + ADCQ 120(SP), R11 + ADCQ 128(SP), R12 + ADCQ 136(SP), R13 + ADCQ 144(SP), R14 + ADCQ 152(SP), R15 + ADCQ 160(SP), AX + ADCQ 168(SP), BX + ADCQ 176(SP), CX + ADCQ 184(SP), DX + MOVQ R8, 96(SP) + MOVQ R9, 104(SP) + MOVQ R10, 112(SP) + MOVQ R11, 120(SP) + MOVQ R12, 128(SP) + MOVQ R13, 136(SP) + MOVQ R14, 144(SP) + MOVQ R15, 152(SP) + MOVQ AX, 160(SP) + MOVQ BX, 168(SP) + MOVQ CX, 176(SP) + MOVQ DX, 184(SP) +// a0b0 + a1b1 stored (96, 184)SP + +// a0 + a1 + MOVQ a+8(FP), DI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + ADDQ 48(DI), R8 + ADCQ 56(DI), R9 + ADCQ 64(DI), R10 + ADCQ 72(DI), R11 + ADCQ 80(DI), R12 + ADCQ 88(DI), R13 + + MOVQ R8, (SP) + MOVQ R9, 8(SP) + MOVQ R10, 16(SP) + MOVQ R11, 24(SP) + MOVQ R12, 32(SP) + MOVQ R13, 40(SP) +// a0 + a1 storeed (0, 40)SP +// b0 + b1 + MOVQ b+16(FP), DI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + ADDQ 48(DI), R8 + ADCQ 56(DI), R9 + ADCQ 64(DI), R10 + ADCQ 72(DI), R11 + ADCQ 80(DI), R12 + ADCQ 88(DI), R13 + + MOVQ R8, 48(SP) + MOVQ R9, 56(SP) + MOVQ R10, 64(SP) + MOVQ R11, 72(SP) + MOVQ R12, 80(SP) + MOVQ R13, 88(SP) +// b0 + b1 storeed (48, 88)SP +// (a0 + a1)(b0 + b1) + +/* i0 */ + XORQ BX, BX + MOVQ (SP), DX + + // | a0 * b0 + MULXQ 48(SP), AX, CX + MOVQ AX, (SP) + + // | a0 * b1 + MULXQ 56(SP), AX, BP + ADCXQ AX, CX + + // | a0 * b2 + MULXQ 64(SP), AX, R9 + ADCXQ AX, BP + + // | a0 * b3 + MULXQ 72(SP), AX, R10 + ADCXQ AX, R9 + + // | a0 * b4 + MULXQ 80(SP), AX, R11 + ADCXQ AX, R10 + + // | a0 * b5 + MULXQ 88(SP), AX, R12 + ADCXQ AX, R11 + ADCXQ BX, R12 + +/* i1 */ + + MOVQ 8(SP), DX + + // | a1 * b0 + MULXQ 48(SP), AX, R13 + ADOXQ AX, CX + ADCXQ R13, BP + MOVQ CX, 8(SP) + + // | a1 * b1 + MULXQ 56(SP), AX, R13 + ADOXQ AX, BP + ADCXQ R13, R9 + + // | a1 * b2 + MULXQ 64(SP), AX, R13 + ADOXQ AX, R9 + ADCXQ R13, R10 + + // | a1 * b3 + MULXQ 72(SP), AX, R13 + ADOXQ AX, R10 + ADCXQ R13, R11 + + // | a1 * b4 + MULXQ 80(SP), AX, R13 + ADOXQ AX, R11 + ADCXQ R13, R12 + + // | a1 * b5 + MULXQ 88(SP), AX, R13 + ADOXQ AX, R12 + ADOXQ BX, R13 + ADCXQ BX, R13 + +/* i2 */ + + MOVQ 16(SP), DX + + // | a2 * b0 + MULXQ 48(SP), AX, R14 + ADOXQ AX, BP + ADCXQ R14, R9 + MOVQ BP, 16(SP) + + // | a2 * b1 + MULXQ 56(SP), AX, R14 + ADOXQ AX, R9 + ADCXQ R14, R10 + + // | a2 * b2 + MULXQ 64(SP), AX, R14 + ADOXQ AX, R10 + ADCXQ R14, R11 + + // | a2 * b3 + MULXQ 72(SP), AX, R14 + ADOXQ AX, R11 + ADCXQ R14, R12 + + // | a2 * b4 + MULXQ 80(SP), AX, R14 + ADOXQ AX, R12 + ADCXQ R14, R13 + + // | a2 * b5 + MULXQ 88(SP), AX, R14 + ADOXQ AX, R13 + ADOXQ BX, R14 + ADCXQ BX, R14 + +/* i3 */ + + MOVQ 24(SP), DX + + // | a3 * b0 + MULXQ 48(SP), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + MOVQ R9, 24(SP) + + // | a3 * b1 + MULXQ 56(SP), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | a3 * b2 + MULXQ 64(SP), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | a3 * b3 + MULXQ 72(SP), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | a3 * b4 + MULXQ 80(SP), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | a3 * b5 + MULXQ 88(SP), AX, R15 + ADOXQ AX, R14 + ADOXQ BX, R15 + ADCXQ BX, R15 + +/* i4 */ + + MOVQ 32(SP), DX + + // | a4 * b0 + MULXQ 48(SP), AX, CX + ADOXQ AX, R10 + ADCXQ CX, R11 + MOVQ R10, 32(SP) + + // | a4 * b1 + MULXQ 56(SP), AX, CX + ADOXQ AX, R11 + ADCXQ CX, R12 + + // | a4 * b2 + MULXQ 64(SP), AX, CX + ADOXQ AX, R12 + ADCXQ CX, R13 + + // | a4 * b3 + MULXQ 72(SP), AX, CX + ADOXQ AX, R13 + ADCXQ CX, R14 + + // | a4 * b4 + MULXQ 80(SP), AX, CX + ADOXQ AX, R14 + ADCXQ CX, R15 + + // | a4 * b5 + MULXQ 88(SP), AX, CX + ADOXQ AX, R15 + ADOXQ BX, CX + ADCXQ BX, CX + +/* i5 */ + + MOVQ 40(SP), DX + + // | a5 * b0 + MULXQ 48(SP), AX, R8 + ADOXQ AX, R11 + ADCXQ R8, R12 + MOVQ R11, 40(SP) + + // | a5 * b1 + MULXQ 56(SP), AX, R8 + ADOXQ AX, R12 + ADCXQ R8, R13 + + // | a5 * b2 + MULXQ 64(SP), AX, R8 + ADOXQ AX, R13 + ADCXQ R8, R14 + + // | a5 * b3 + MULXQ 72(SP), AX, R8 + ADOXQ AX, R14 + ADCXQ R8, R15 + + // | a5 * b4 + MULXQ 80(SP), AX, R8 + ADOXQ AX, R15 + ADCXQ R8, CX + + // | a5 * b5 + MULXQ 88(SP), AX, R8 + ADOXQ AX, CX + ADOXQ BX, R8 + ADCXQ BX, R8 + + MOVQ R12, 48(SP) + MOVQ R13, 56(SP) + MOVQ R14, 64(SP) + MOVQ R15, 72(SP) + MOVQ CX, 80(SP) + MOVQ R8, 88(SP) + + +// (a0 + a1)(b0 + b1) - (a0b0 + a1b1) + MOVQ (SP), R8 + MOVQ 8(SP), R9 + MOVQ 16(SP), R10 + MOVQ 24(SP), R11 + MOVQ 32(SP), R12 + MOVQ 40(SP), R13 + MOVQ 48(SP), R14 + MOVQ 56(SP), R15 + MOVQ 64(SP), AX + MOVQ 72(SP), BX + MOVQ 80(SP), CX + MOVQ 88(SP), DX + + SUBQ 96(SP), R8 + SBBQ 104(SP), R9 + SBBQ 112(SP), R10 + SBBQ 120(SP), R11 + SBBQ 128(SP), R12 + SBBQ 136(SP), R13 + SBBQ 144(SP), R14 + SBBQ 152(SP), R15 + SBBQ 160(SP), AX + SBBQ 168(SP), BX + SBBQ 176(SP), CX + SBBQ 184(SP), DX + + MOVQ c+0(FP), DI + MOVQ R8, 96(DI) + MOVQ R9, 104(DI) + MOVQ R10, 112(DI) + MOVQ R11, 120(DI) + MOVQ R12, 128(DI) + MOVQ R13, 136(DI) + MOVQ R14, 144(DI) + MOVQ R15, 152(DI) + MOVQ AX, 160(DI) + MOVQ BX, 168(DI) + MOVQ CX, 176(DI) + MOVQ DX, 184(DI) + + RET diff --git a/crypto/bls12381/fp6.go b/crypto/bls12381/fp6.go index 304173baa3..509b0ec56d 100644 --- a/crypto/bls12381/fp6.go +++ b/crypto/bls12381/fp6.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( @@ -22,7 +6,8 @@ import ( ) type fp6Temp struct { - t [6]*fe2 + t [5]*fe2 + wt [6]*wfe2 } type fp6 struct { @@ -31,11 +16,15 @@ type fp6 struct { } func newFp6Temp() fp6Temp { - t := [6]*fe2{} + t := [5]*fe2{} for i := 0; i < len(t); i++ { t[i] = &fe2{} } - return fp6Temp{t} + wt := [6]*wfe2{} + for i := 0; i < len(wt); i++ { + wt[i] = &wfe2{} + } + return fp6Temp{t, wt} } func newFp6(f *fp2) *fp6 { @@ -47,19 +36,19 @@ func newFp6(f *fp2) *fp6 { } func (e *fp6) fromBytes(b []byte) (*fe6, error) { - if len(b) < 288 { - return nil, errors.New("input string should be larger than 288 bytes") + if len(b) != 288 { + return nil, errors.New("input string length must be equal to 288 bytes") } fp2 := e.fp2 - u2, err := fp2.fromBytes(b[:96]) + u2, err := fp2.fromBytes(b[:2*fpByteSize]) if err != nil { return nil, err } - u1, err := fp2.fromBytes(b[96:192]) + u1, err := fp2.fromBytes(b[2*fpByteSize : 4*fpByteSize]) if err != nil { return nil, err } - u0, err := fp2.fromBytes(b[192:]) + u0, err := fp2.fromBytes(b[4*fpByteSize:]) if err != nil { return nil, err } @@ -68,10 +57,10 @@ func (e *fp6) fromBytes(b []byte) (*fe6, error) { func (e *fp6) toBytes(a *fe6) []byte { fp2 := e.fp2 - out := make([]byte, 288) - copy(out[:96], fp2.toBytes(&a[2])) - copy(out[96:192], fp2.toBytes(&a[1])) - copy(out[192:], fp2.toBytes(&a[0])) + out := make([]byte, 6*fpByteSize) + copy(out[:2*fpByteSize], fp2.toBytes(&a[2])) + copy(out[2*fpByteSize:4*fpByteSize], fp2.toBytes(&a[1])) + copy(out[4*fpByteSize:], fp2.toBytes(&a[0])) return out } @@ -87,188 +76,368 @@ func (e *fp6) one() *fe6 { return new(fe6).one() } -func (e *fp6) add(c, a, b *fe6) { - fp2 := e.fp2 - fp2.add(&c[0], &a[0], &b[0]) - fp2.add(&c[1], &a[1], &b[1]) - fp2.add(&c[2], &a[2], &b[2]) +func fp6Ladd(c, a, b *fe6) { + fp2Ladd(&c[0], &a[0], &b[0]) + fp2Ladd(&c[1], &a[1], &b[1]) + fp2Ladd(&c[2], &a[2], &b[2]) } -func (e *fp6) addAssign(a, b *fe6) { - fp2 := e.fp2 - fp2.addAssign(&a[0], &b[0]) - fp2.addAssign(&a[1], &b[1]) - fp2.addAssign(&a[2], &b[2]) +func wfp6SubAssign(a, b *wfe6) { + wfp2SubAssign(&a[0], &b[0]) + wfp2SubAssign(&a[1], &b[1]) + wfp2SubAssign(&a[2], &b[2]) } -func (e *fp6) double(c, a *fe6) { - fp2 := e.fp2 - fp2.double(&c[0], &a[0]) - fp2.double(&c[1], &a[1]) - fp2.double(&c[2], &a[2]) +func wfp6AddAssign(a, b *wfe6) { + wfp2AddAssign(&a[0], &b[0]) + wfp2AddAssign(&a[1], &b[1]) + wfp2AddAssign(&a[2], &b[2]) } -func (e *fp6) doubleAssign(a *fe6) { - fp2 := e.fp2 - fp2.doubleAssign(&a[0]) - fp2.doubleAssign(&a[1]) - fp2.doubleAssign(&a[2]) +func fp6Add(c, a, b *fe6) { + fp2Add(&c[0], &a[0], &b[0]) + fp2Add(&c[1], &a[1], &b[1]) + fp2Add(&c[2], &a[2], &b[2]) } -func (e *fp6) sub(c, a, b *fe6) { - fp2 := e.fp2 - fp2.sub(&c[0], &a[0], &b[0]) - fp2.sub(&c[1], &a[1], &b[1]) - fp2.sub(&c[2], &a[2], &b[2]) +func fp6AddAssign(a, b *fe6) { + fp2AddAssign(&a[0], &b[0]) + fp2AddAssign(&a[1], &b[1]) + fp2AddAssign(&a[2], &b[2]) } -func (e *fp6) subAssign(a, b *fe6) { - fp2 := e.fp2 - fp2.subAssign(&a[0], &b[0]) - fp2.subAssign(&a[1], &b[1]) - fp2.subAssign(&a[2], &b[2]) +func fp6Double(c, a *fe6) { + fp2Double(&c[0], &a[0]) + fp2Double(&c[1], &a[1]) + fp2Double(&c[2], &a[2]) } -func (e *fp6) neg(c, a *fe6) { - fp2 := e.fp2 - fp2.neg(&c[0], &a[0]) - fp2.neg(&c[1], &a[1]) - fp2.neg(&c[2], &a[2]) +func fp6DoubleAssign(a *fe6) { + fp2DoubleAssign(&a[0]) + fp2DoubleAssign(&a[1]) + fp2DoubleAssign(&a[2]) } -func (e *fp6) mul(c, a, b *fe6) { - fp2, t := e.fp2, e.t - fp2.mul(t[0], &a[0], &b[0]) - fp2.mul(t[1], &a[1], &b[1]) - fp2.mul(t[2], &a[2], &b[2]) - fp2.add(t[3], &a[1], &a[2]) - fp2.add(t[4], &b[1], &b[2]) - fp2.mulAssign(t[3], t[4]) - fp2.add(t[4], t[1], t[2]) - fp2.subAssign(t[3], t[4]) - fp2.mulByNonResidue(t[3], t[3]) - fp2.add(t[5], t[0], t[3]) - fp2.add(t[3], &a[0], &a[1]) - fp2.add(t[4], &b[0], &b[1]) - fp2.mulAssign(t[3], t[4]) - fp2.add(t[4], t[0], t[1]) - fp2.subAssign(t[3], t[4]) - fp2.mulByNonResidue(t[4], t[2]) - fp2.add(&c[1], t[3], t[4]) - fp2.add(t[3], &a[0], &a[2]) - fp2.add(t[4], &b[0], &b[2]) - fp2.mulAssign(t[3], t[4]) - fp2.add(t[4], t[0], t[2]) - fp2.subAssign(t[3], t[4]) - fp2.add(&c[2], t[1], t[3]) - c[0].set(t[5]) +func fp6Sub(c, a, b *fe6) { + fp2Sub(&c[0], &a[0], &b[0]) + fp2Sub(&c[1], &a[1], &b[1]) + fp2Sub(&c[2], &a[2], &b[2]) +} + +func fp6SubAssign(a, b *fe6) { + fp2SubAssign(&a[0], &b[0]) + fp2SubAssign(&a[1], &b[1]) + fp2SubAssign(&a[2], &b[2]) +} + +func fp6Neg(c, a *fe6) { + fp2Neg(&c[0], &a[0]) + fp2Neg(&c[1], &a[1]) + fp2Neg(&c[2], &a[2]) +} + +func (e *fp6) wmul01(c *wfe6, a *fe6, b0, b1 *fe2) { + wt, t := e.wt, e.t + wfp2Mul(wt[0], &a[0], b0) // v0 = b0a0 + wfp2Mul(wt[1], &a[1], b1) // v1 = a1b1 + fp2Ladd(t[2], &a[1], &a[2]) // a1 + a2 + wfp2Mul(wt[2], t[2], b1) // b1(a1 + a2) + wfp2SubAssign(wt[2], wt[1]) // b1(a1 + a2) - v1 + wfp2MulByNonResidueAssign(wt[2]) + fp2Ladd(t[3], &a[0], &a[2]) // a0 + a2 + wfp2Mul(wt[3], t[3], b0) // b0(a0 + a2) + wfp2SubAssign(wt[3], wt[0]) + wfp2Add(&c[2], wt[3], wt[1]) + fp2Ladd(t[0], b0, b1) // (b0 + b1) + fp2Ladd(t[1], &a[0], &a[1]) // (a0 + a1) + wfp2Mul(wt[4], t[0], t[1]) // (a0 + a1)(b0 + b1) + wfp2SubAssign(wt[4], wt[0]) + wfp2Sub(&c[1], wt[4], wt[1]) + wfp2Add(&c[0], wt[2], wt[0]) +} + +func (e *fp6) wmul1(c *wfe6, a *fe6, b1 *fe2) { + wt := e.wt + wfp2Mul(wt[0], &a[2], b1) + wfp2Mul(&c[2], &a[1], b1) + wfp2Mul(&c[1], &a[0], b1) + wfp2MulByNonResidue(&c[0], wt[0]) +} + +func (e *fp6) wmul(c *wfe6, a, b *fe6) { + + wt, t := e.wt, e.t + + // Faster Explicit Formulas for Computing Pairings over Ordinary Curves + // AKLGL + // https://eprint.iacr.org/2010/526.pdf + // Algorithm 3 + + // 1. T0 = a0b0,T1 = a1b1, T2 = a2b2 + wfp2Mul(wt[0], &a[0], &b[0]) + wfp2Mul(wt[1], &a[1], &b[1]) + wfp2Mul(wt[2], &a[2], &b[2]) + // 2. t0 = a1 + a2, t1 = b1 + b2 + fp2Ladd(t[0], &a[1], &a[2]) + fp2Ladd(t[1], &b[1], &b[2]) + // 3. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + // 4. T4 = T1 + T2 + wfp2Add(wt[4], wt[1], wt[2]) + + // 5,6. T3 = T3 - T4 + wfp2SubMixedAssign(wt[3], wt[4]) + + // 7. T4 = β * T3 + wfp2MulByNonResidue(wt[4], wt[3]) + + // 8. T5 = T4 + T0 + wfp2Add(wt[5], wt[4], wt[0]) + + // 9. t0 = a0 + a1, t1 = b0 + b1 + fp2Ladd(t[0], &a[0], &a[1]) + fp2Ladd(t[1], &b[0], &b[1]) + + // 10. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + + // 11. T4 = T0 + T1 + wfp2Add(wt[4], wt[0], wt[1]) + + // 12,13. T3 = T3 - T4 + wfp2SubMixedAssign(wt[3], wt[4]) + + // 14,15. T4 = β * T2 + wfp2MulByNonResidue(wt[4], wt[2]) + + // 17. t0 = a0 + a2, t1 = b0 + b2 + fp2Ladd(t[0], &a[0], &a[2]) + fp2Ladd(t[1], &b[0], &b[2]) + + // 16. T6 = T3 + T4 + wfp2Add(&c[1], wt[3], wt[4]) + + // 18. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + + // 19. T4 = T0 + T2 + wfp2Add(wt[4], wt[0], wt[2]) + + // 20,21. T3 = T3 - T4 + wfp2SubMixedAssign(wt[3], wt[4]) + + // 22,23. T7 = T3 + T1 + wfp2AddMixed(&c[2], wt[3], wt[1]) + + // c = T5, T6, T7 + c[0].set(wt[5]) +} + +func (e *fp6) mul(c *fe6, a, b *fe6) { + wt, t := e.wt, e.t + + // 1. T0 = a0b0,T1 = a1b1, T2 = a2b2 + wfp2Mul(wt[0], &a[0], &b[0]) + wfp2Mul(wt[1], &a[1], &b[1]) + wfp2Mul(wt[2], &a[2], &b[2]) + // 2. t0 = a1 + a2, t1 = b1 + b2 + fp2Ladd(t[0], &a[1], &a[2]) + fp2Ladd(t[1], &b[1], &b[2]) + // 3. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + // 4. T4 = T1 + T2 + wfp2Add(wt[4], wt[1], wt[2]) + + // 5,6. T3 = T3 - T4 + wfp2SubMixedAssign(wt[3], wt[4]) + + // 7. T4 = β * T3 + wfp2MulByNonResidue(wt[4], wt[3]) + + // 8. T5 = T4 + T0 + wfp2Add(wt[5], wt[4], wt[0]) + + // 9. t0 = a0 + a1, t1 = b0 + b1 + fp2Ladd(t[0], &a[0], &a[1]) + fp2Ladd(t[1], &b[0], &b[1]) + + // 10. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + + // 11. T4 = T0 + T1 + wfp2Add(wt[4], wt[0], wt[1]) + + // 12,13. T3 = T3 - T4 + wfp2SubMixed(wt[3], wt[3], wt[4]) + + // 14,15. T4 = β * T2 + wfp2MulByNonResidue(wt[4], wt[2]) + + // 17. t0 = a0 + a2, t1 = b0 + b2 + fp2Ladd(t[0], &a[0], &a[2]) + fp2Ladd(t[1], &b[0], &b[2]) + + // 16. T6 = T3 + T4 + wfp2Add(wt[3], wt[3], wt[4]) + c[1].fromWide(wt[3]) + + // 18. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + + // 19. T4 = T0 + T2 + wfp2Add(wt[4], wt[0], wt[2]) + + // 20,21. T3 = T3 - T4 + wfp2SubMixed(wt[3], wt[3], wt[4]) + + // 22,23. T7 = T3 + T1 + wfp2AddMixed(wt[3], wt[3], wt[1]) + c[2].fromWide(wt[3]) + + // c = T5, T6, T7 + c[0].fromWide(wt[5]) } func (e *fp6) mulAssign(a, b *fe6) { - fp2, t := e.fp2, e.t - fp2.mul(t[0], &a[0], &b[0]) - fp2.mul(t[1], &a[1], &b[1]) - fp2.mul(t[2], &a[2], &b[2]) - fp2.add(t[3], &a[1], &a[2]) - fp2.add(t[4], &b[1], &b[2]) - fp2.mulAssign(t[3], t[4]) - fp2.add(t[4], t[1], t[2]) - fp2.subAssign(t[3], t[4]) - fp2.mulByNonResidue(t[3], t[3]) - fp2.add(t[5], t[0], t[3]) - fp2.add(t[3], &a[0], &a[1]) - fp2.add(t[4], &b[0], &b[1]) - fp2.mulAssign(t[3], t[4]) - fp2.add(t[4], t[0], t[1]) - fp2.subAssign(t[3], t[4]) - fp2.mulByNonResidue(t[4], t[2]) - fp2.add(&a[1], t[3], t[4]) - fp2.add(t[3], &a[0], &a[2]) - fp2.add(t[4], &b[0], &b[2]) - fp2.mulAssign(t[3], t[4]) - fp2.add(t[4], t[0], t[2]) - fp2.subAssign(t[3], t[4]) - fp2.add(&a[2], t[1], t[3]) - a[0].set(t[5]) + wt, t := e.wt, e.t + + // Faster Explicit Formulas for Computing Pairings over Ordinary Curves + // AKLGL + // https://eprint.iacr.org/2010/526.pdf + // Algorithm 3 + + // 1. T0 = a0b0,T1 = a1b1, T2 = a2b2 + wfp2Mul(wt[0], &a[0], &b[0]) + wfp2Mul(wt[1], &a[1], &b[1]) + wfp2Mul(wt[2], &a[2], &b[2]) + // 2. t0 = a1 + a2, t1 = b1 + b2 + fp2Ladd(t[0], &a[1], &a[2]) + fp2Ladd(t[1], &b[1], &b[2]) + // 3. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + // 4. T4 = T1 + T2 + wfp2Add(wt[4], wt[1], wt[2]) + + // 5,6. T3 = T3 - T4 + wfp2SubMixed(wt[3], wt[3], wt[4]) + + // 7. T4 = β * T3 + wfp2MulByNonResidue(wt[4], wt[3]) + + // 8. T5 = T4 + T0 + wfp2Add(wt[5], wt[4], wt[0]) + + // 9. t0 = a0 + a1, t1 = b0 + b1 + fp2Ladd(t[0], &a[0], &a[1]) + fp2Ladd(t[1], &b[0], &b[1]) + + // 10. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + + // 11. T4 = T0 + T1 + wfp2Add(wt[4], wt[0], wt[1]) + + // 12,13. T3 = T3 - T4 + wfp2SubMixed(wt[3], wt[3], wt[4]) + + // 14,15. T4 = β * T2 + wfp2MulByNonResidue(wt[4], wt[2]) + + // 17. t0 = a0 + a2, t1 = b0 + b2 + fp2Ladd(t[0], &a[0], &a[2]) + fp2Ladd(t[1], &b[0], &b[2]) + + // 16. T6 = T3 + T4 + wfp2Add(wt[3], wt[3], wt[4]) + a[1].fromWide(wt[3]) + + // 18. T3 = t0 * t1 + wfp2Mul(wt[3], t[0], t[1]) + + // 19. T4 = T0 + T2 + wfp2Add(wt[4], wt[0], wt[2]) + + // 20,21. T3 = T3 - T4 + wfp2SubMixed(wt[3], wt[3], wt[4]) + + // 22,23. T7 = T3 + T1 + wfp2AddMixed(wt[3], wt[3], wt[1]) + a[2].fromWide(wt[3]) + + // a = T5, T6, T7 + a[0].fromWide(wt[5]) } func (e *fp6) square(c, a *fe6) { - fp2, t := e.fp2, e.t - fp2.square(t[0], &a[0]) - fp2.mul(t[1], &a[0], &a[1]) - fp2.doubleAssign(t[1]) - fp2.sub(t[2], &a[0], &a[1]) - fp2.addAssign(t[2], &a[2]) - fp2.squareAssign(t[2]) - fp2.mul(t[3], &a[1], &a[2]) - fp2.doubleAssign(t[3]) - fp2.square(t[4], &a[2]) - fp2.mulByNonResidue(t[5], t[3]) - fp2.add(&c[0], t[0], t[5]) - fp2.mulByNonResidue(t[5], t[4]) - fp2.add(&c[1], t[1], t[5]) - fp2.addAssign(t[1], t[2]) - fp2.addAssign(t[1], t[3]) - fp2.addAssign(t[0], t[4]) - fp2.sub(&c[2], t[1], t[0]) + wt, t := e.wt, e.t + wfp2Square(wt[0], &a[0]) + wfp2Mul(wt[1], &a[0], &a[1]) + wfp2DoubleAssign(wt[1]) + fp2Sub(t[2], &a[0], &a[1]) + fp2AddAssign(t[2], &a[2]) + wfp2Square(wt[2], t[2]) + wfp2Mul(wt[3], &a[1], &a[2]) + wfp2DoubleAssign(wt[3]) + wfp2Square(wt[4], &a[2]) + wfp2MulByNonResidue(wt[5], wt[3]) + wfp2AddAssign(wt[5], wt[0]) + c[0].fromWide(wt[5]) + wfp2MulByNonResidue(wt[5], wt[4]) + wfp2AddAssign(wt[5], wt[1]) + c[1].fromWide(wt[5]) + wfp2AddAssign(wt[1], wt[2]) + wfp2AddAssign(wt[1], wt[3]) + wfp2AddAssign(wt[0], wt[4]) + wfp2SubAssign(wt[1], wt[0]) + c[2].fromWide(wt[1]) + } -func (e *fp6) mulBy01Assign(a *fe6, b0, b1 *fe2) { - fp2, t := e.fp2, e.t - fp2.mul(t[0], &a[0], b0) - fp2.mul(t[1], &a[1], b1) - fp2.add(t[5], &a[1], &a[2]) - fp2.mul(t[2], b1, t[5]) - fp2.subAssign(t[2], t[1]) - fp2.mulByNonResidue(t[2], t[2]) - fp2.add(t[5], &a[0], &a[2]) - fp2.mul(t[3], b0, t[5]) - fp2.subAssign(t[3], t[0]) - fp2.add(&a[2], t[3], t[1]) - fp2.add(t[4], b0, b1) - fp2.add(t[5], &a[0], &a[1]) - fp2.mulAssign(t[4], t[5]) - fp2.subAssign(t[4], t[0]) - fp2.sub(&a[1], t[4], t[1]) - fp2.add(&a[0], t[2], t[0]) -} - -func (e *fp6) mulBy01(c, a *fe6, b0, b1 *fe2) { - fp2, t := e.fp2, e.t - fp2.mul(t[0], &a[0], b0) - fp2.mul(t[1], &a[1], b1) - fp2.add(t[2], &a[1], &a[2]) - fp2.mulAssign(t[2], b1) - fp2.subAssign(t[2], t[1]) - fp2.mulByNonResidue(t[2], t[2]) - fp2.add(t[3], &a[0], &a[2]) - fp2.mulAssign(t[3], b0) - fp2.subAssign(t[3], t[0]) - fp2.add(&c[2], t[3], t[1]) - fp2.add(t[4], b0, b1) - fp2.add(t[3], &a[0], &a[1]) - fp2.mulAssign(t[4], t[3]) - fp2.subAssign(t[4], t[0]) - fp2.sub(&c[1], t[4], t[1]) - fp2.add(&c[0], t[2], t[0]) -} - -func (e *fp6) mulBy1(c, a *fe6, b1 *fe2) { - fp2, t := e.fp2, e.t - fp2.mul(t[0], &a[2], b1) - fp2.mul(&c[2], &a[1], b1) - fp2.mul(&c[1], &a[0], b1) - fp2.mulByNonResidue(&c[0], t[0]) +func (e *fp6) wsquare(c *wfe6, a *fe6) { + wt, t := e.wt, e.t + wfp2Square(wt[0], &a[0]) + wfp2Mul(wt[1], &a[0], &a[1]) + wfp2DoubleAssign(wt[1]) + fp2Sub(t[2], &a[0], &a[1]) + fp2AddAssign(t[2], &a[2]) + wfp2Square(wt[2], t[2]) + wfp2Mul(wt[3], &a[1], &a[2]) + wfp2DoubleAssign(wt[3]) + wfp2Square(wt[4], &a[2]) + wfp2MulByNonResidue(wt[5], wt[3]) + wfp2Add(&c[0], wt[5], wt[0]) + wfp2MulByNonResidue(wt[5], wt[4]) + wfp2Add(&c[1], wt[1], wt[5]) + wfp2AddAssign(wt[1], wt[2]) + wfp2AddAssign(wt[1], wt[3]) + wfp2AddAssign(wt[0], wt[4]) + wfp2Sub(&c[2], wt[1], wt[0]) } func (e *fp6) mulByNonResidue(c, a *fe6) { - fp2, t := e.fp2, e.t + t := e.t t[0].set(&a[0]) - fp2.mulByNonResidue(&c[0], &a[2]) + mulByNonResidue(&c[0], &a[2]) c[2].set(&a[1]) c[1].set(t[0]) } +func (e *fp6) wmulByNonResidue(c, a *wfe6) { + t := e.wt + t[0].set(&a[0]) + wfp2MulByNonResidue(&c[0], &a[2]) + c[2].set(&a[1]) + c[1].set(t[0]) +} + +func (e *fp6) wmulByNonResidueAssign(a *wfe6) { + t := e.wt + t[0].set(&a[0]) + wfp2MulByNonResidue(&a[0], &a[2]) + a[2].set(&a[1]) + a[1].set(t[0]) +} + func (e *fp6) mulByBaseField(c, a *fe6, b *fe2) { fp2 := e.fp2 fp2.mul(&c[0], &a[0], b) @@ -291,61 +460,57 @@ func (e *fp6) inverse(c, a *fe6) { fp2, t := e.fp2, e.t fp2.square(t[0], &a[0]) fp2.mul(t[1], &a[1], &a[2]) - fp2.mulByNonResidue(t[1], t[1]) - fp2.subAssign(t[0], t[1]) - fp2.square(t[1], &a[1]) - fp2.mul(t[2], &a[0], &a[2]) - fp2.subAssign(t[1], t[2]) - fp2.square(t[2], &a[2]) - fp2.mulByNonResidue(t[2], t[2]) - fp2.mul(t[3], &a[0], &a[1]) - fp2.subAssign(t[2], t[3]) - fp2.mul(t[3], &a[2], t[2]) - fp2.mul(t[4], &a[1], t[1]) - fp2.addAssign(t[3], t[4]) - fp2.mulByNonResidue(t[3], t[3]) - fp2.mul(t[4], &a[0], t[0]) - fp2.addAssign(t[3], t[4]) - fp2.inverse(t[3], t[3]) - fp2.mul(&c[0], t[0], t[3]) - fp2.mul(&c[1], t[2], t[3]) - fp2.mul(&c[2], t[1], t[3]) + mulByNonResidueAssign(t[1]) + fp2SubAssign(t[0], t[1]) // A = v0 - βv5 + fp2.square(t[1], &a[1]) // v1 = a1^2 + fp2.mul(t[2], &a[0], &a[2]) // v4 = a0a2 + fp2SubAssign(t[1], t[2]) // C = v1 - v4 + fp2.square(t[2], &a[2]) // v2 = a2^2 + mulByNonResidueAssign(t[2]) // βv2 + fp2.mul(t[3], &a[0], &a[1]) // v3 = a0a1 + fp2SubAssign(t[2], t[3]) // B = βv2 - v3 + fp2.mul(t[3], &a[2], t[2]) // B * a2 + fp2.mul(t[4], &a[1], t[1]) // C * a1 + fp2AddAssign(t[3], t[4]) // Ca1 + Ba2 + mulByNonResidueAssign(t[3]) // β(Ca1 + Ba2) + fp2.mul(t[4], &a[0], t[0]) // Aa0 + fp2AddAssign(t[3], t[4]) // v6 = Aa0 + β(Ca1 + Ba2) + fp2.inverse(t[3], t[3]) // F = v6^-1 + fp2.mul(&c[0], t[0], t[3]) // c0 = AF + fp2.mul(&c[1], t[2], t[3]) // c1 = BF + fp2.mul(&c[2], t[1], t[3]) // c2 = CF } -func (e *fp6) frobeniusMap(c, a *fe6, power uint) { +func (e *fp6) frobeniusMap(a *fe6, power int) { fp2 := e.fp2 - fp2.frobeniusMap(&c[0], &a[0], power) - fp2.frobeniusMap(&c[1], &a[1], power) - fp2.frobeniusMap(&c[2], &a[2], power) - switch power % 6 { - case 0: - return - case 3: - neg(&c[0][0], &a[1][1]) - c[1][1].set(&a[1][0]) - fp2.neg(&a[2], &a[2]) - default: - fp2.mul(&c[1], &c[1], &frobeniusCoeffs61[power%6]) - fp2.mul(&c[2], &c[2], &frobeniusCoeffs62[power%6]) - } + fp2.frobeniusMap(&a[0], power) + fp2.frobeniusMap(&a[1], power) + fp2.frobeniusMap(&a[2], power) + fp2.mulAssign(&a[1], &frobeniusCoeffs61[power%6]) + fp2.mulAssign(&a[2], &frobeniusCoeffs62[power%6]) } -func (e *fp6) frobeniusMapAssign(a *fe6, power uint) { +func (e *fp6) frobeniusMap1(a *fe6) { fp2 := e.fp2 - fp2.frobeniusMapAssign(&a[0], power) - fp2.frobeniusMapAssign(&a[1], power) - fp2.frobeniusMapAssign(&a[2], power) + fp2.frobeniusMap1(&a[0]) + fp2.frobeniusMap1(&a[1]) + fp2.frobeniusMap1(&a[2]) + fp2.mulAssign(&a[1], &frobeniusCoeffs61[1]) + fp2.mulAssign(&a[2], &frobeniusCoeffs62[1]) +} + +func (e *fp6) frobeniusMap2(a *fe6) { + e.fp2.mulAssign(&a[1], &frobeniusCoeffs61[2]) + e.fp2.mulAssign(&a[2], &frobeniusCoeffs62[2]) +} + +func (e *fp6) frobeniusMap3(a *fe6) { t := e.t - switch power % 6 { - case 0: - return - case 3: - neg(&t[0][0], &a[1][1]) - a[1][1].set(&a[1][0]) - a[1][0].set(&t[0][0]) - fp2.neg(&a[2], &a[2]) - default: - fp2.mulAssign(&a[1], &frobeniusCoeffs61[power%6]) - fp2.mulAssign(&a[2], &frobeniusCoeffs62[power%6]) - } + e.fp2.frobeniusMap1(&a[0]) + e.fp2.frobeniusMap1(&a[1]) + e.fp2.frobeniusMap1(&a[2]) + neg(&t[0][0], &a[1][1]) + a[1][1].set(&a[1][0]) + a[1][0].set(&t[0][0]) + fp2Neg(&a[2], &a[2]) } diff --git a/crypto/bls12381/fp_arithmetic_x86.s b/crypto/bls12381/fp_arithmetic_x86.s new file mode 100644 index 0000000000..04abfe5fe0 --- /dev/null +++ b/crypto/bls12381/fp_arithmetic_x86.s @@ -0,0 +1,3894 @@ +// +build amd64,!generic + +#include "textflag.h" +#include "funcdata.h" + +// assigned addition with modular reduction +// a = (a + b) % p +TEXT ·addAssign(SB), NOSPLIT, $0-16 + // | + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // | + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + + // | + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, SI + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + // | + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET + +/* | end */ + + +// addition wth modular reduction +// c = (a + b) % p +TEXT ·add(SB), NOSPLIT, $0-24 + // | + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // | + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + + // | + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + MOVQ $0xb9feffffffffaaab, DI + SUBQ DI, R14 + MOVQ $0x1eabfffeb153ffff, DI + SBBQ DI, R15 + MOVQ $0x6730d2a0f6b0f624, DI + SBBQ DI, CX + MOVQ $0x64774b84f38512bf, DI + SBBQ DI, DX + MOVQ $0x4b1ba7b6434bacd7, DI + SBBQ DI, SI + MOVQ $0x1a0111ea397fe69a, DI + SBBQ DI, BX + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + // | + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// addition w/o modular reduction +// c = a + b +TEXT ·ladd(SB), NOSPLIT, $0-24 + // | + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // | + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + + // | + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// assigned addition w/o modular reduction +// a = a + b +TEXT ·laddAssign(SB), NOSPLIT, $0-16 + // | + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // | + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + + // | + MOVQ a+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// subtraction with modular reduction +// c = (a - b) % p +TEXT ·sub(SB), NOSPLIT, $0-24 + // | + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + XORQ AX, AX + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + + // | + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, SI + MOVQ $0x1a0111ea397fe69a, BX + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, SI + CMOVQCC AX, BX + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ SI, R12 + ADCQ BX, R13 + + // | + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// assigned subtraction with modular reduction +// a' = (a - b) % p +TEXT ·subAssign(SB), NOSPLIT, $0-16 + // | + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + XORQ AX, AX + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + + // | + MOVQ $0xb9feffffffffaaab, R14 + MOVQ $0x1eabfffeb153ffff, R15 + MOVQ $0x6730d2a0f6b0f624, CX + MOVQ $0x64774b84f38512bf, DX + MOVQ $0x4b1ba7b6434bacd7, SI + MOVQ $0x1a0111ea397fe69a, BX + CMOVQCC AX, R14 + CMOVQCC AX, R15 + CMOVQCC AX, CX + CMOVQCC AX, DX + CMOVQCC AX, SI + CMOVQCC AX, BX + ADDQ R14, R8 + ADCQ R15, R9 + ADCQ CX, R10 + ADCQ DX, R11 + ADCQ SI, R12 + ADCQ BX, R13 + + // | + MOVQ a+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// assigned subtraction without modular reduction +// a = a - b +TEXT ·lsubAssign(SB), NOSPLIT, $0-16 + // | + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + + // | + MOVQ a+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// doubling with modular reduction +// c = (2 * a) % p +TEXT ·double(SB), NOSPLIT, $0-16 + // | + MOVQ a+8(FP), DI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + + // | + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + MOVQ $0xb9feffffffffaaab, DI + SUBQ DI, R14 + MOVQ $0x1eabfffeb153ffff, DI + SBBQ DI, R15 + MOVQ $0x6730d2a0f6b0f624, DI + SBBQ DI, CX + MOVQ $0x64774b84f38512bf, DI + SBBQ DI, DX + MOVQ $0x4b1ba7b6434bacd7, DI + SBBQ DI, SI + MOVQ $0x1a0111ea397fe69a, DI + SBBQ DI, BX + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + // | + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// assigned doubling with modular reduction +// a = (2 * a) % p +TEXT ·doubleAssign(SB), NOSPLIT, $0-8 + // | + MOVQ a+0(FP), DI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + + // | + MOVQ R8, R14 + MOVQ R9, R15 + MOVQ R10, CX + MOVQ R11, DX + MOVQ R12, SI + MOVQ R13, BX + MOVQ $0xb9feffffffffaaab, AX + SUBQ AX, R14 + MOVQ $0x1eabfffeb153ffff, AX + SBBQ AX, R15 + MOVQ $0x6730d2a0f6b0f624, AX + SBBQ AX, CX + MOVQ $0x64774b84f38512bf, AX + SBBQ AX, DX + MOVQ $0x4b1ba7b6434bacd7, AX + SBBQ AX, SI + MOVQ $0x1a0111ea397fe69a, AX + SBBQ AX, BX + CMOVQCC R14, R8 + CMOVQCC R15, R9 + CMOVQCC CX, R10 + CMOVQCC DX, R11 + CMOVQCC SI, R12 + CMOVQCC BX, R13 + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// doubling without modular reduction +// c = 2 * a +TEXT ·ldouble(SB), NOSPLIT, $0-16 + // | + MOVQ a+8(FP), DI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + + // | + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + + // | + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + RET +/* | end */ + + +TEXT ·_neg(SB), NOSPLIT, $0-16 + // | + MOVQ a+8(FP), DI + + // | + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + SUBQ (DI), R8 + SBBQ 8(DI), R9 + SBBQ 16(DI), R10 + SBBQ 24(DI), R11 + SBBQ 32(DI), R12 + SBBQ 40(DI), R13 + + // | + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + RET +/* | end */ + + +// multiplication without using MULX/ADX +// c = a * b % p +TEXT ·mulNoADX(SB), NOSPLIT, $24-24 + // | + +/* inputs */ + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ $0x00, R9 + MOVQ $0x00, R10 + MOVQ $0x00, R11 + MOVQ $0x00, R12 + MOVQ $0x00, R13 + MOVQ $0x00, R14 + MOVQ $0x00, R15 + + // | + +/* i0 */ + + // | a0 @ CX + MOVQ (DI), CX + + // | a0 * b0 + MOVQ (SI), AX + MULQ CX + MOVQ AX, (SP) + MOVQ DX, R8 + + // | a0 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R8 + ADCQ DX, R9 + + // | a0 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + + // | a0 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + + // | a0 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + + // | a0 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + + // | + +/* i1 */ + + // | a1 @ CX + MOVQ 8(DI), CX + MOVQ $0x00, BX + + // | a1 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R8 + ADCQ DX, R9 + ADCQ $0x00, R10 + ADCQ $0x00, BX + MOVQ R8, 8(SP) + MOVQ $0x00, R8 + + // | a1 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + ADCQ BX, R11 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a1 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ BX, R12 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a1 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a1 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + + // | a1 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + + // | + +/* i2 */ + + // | a2 @ CX + MOVQ 16(DI), CX + MOVQ $0x00, BX + + // | a2 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + ADCQ $0x00, R11 + ADCQ $0x00, BX + MOVQ R9, 16(SP) + MOVQ $0x00, R9 + + // | a2 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ BX, R12 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a2 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a2 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a2 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ BX, R15 + + // | a2 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, R15 + + // | + +/* i3 */ + + // | a3 @ CX + MOVQ 24(DI), CX + MOVQ $0x00, BX + + // | a3 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ $0x00, R12 + ADCQ $0x00, BX + + // | a3 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a3 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a3 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ BX, R15 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a3 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, R15 + ADCQ BX, R8 + + // | a3 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R15 + ADCQ DX, R8 + + // | + +/* i4 */ + + // | a4 @ CX + MOVQ 32(DI), CX + MOVQ $0x00, BX + + // | a4 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ $0x00, R13 + ADCQ $0x00, BX + + // | a4 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a4 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ BX, R15 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a4 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, R15 + ADCQ BX, R8 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a4 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R15 + ADCQ DX, R8 + ADCQ BX, R9 + + // | a4 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R8 + ADCQ DX, R9 + + // | + +/* i5 */ + + // | a5 @ CX + MOVQ 40(DI), CX + MOVQ $0x00, BX + + // | a5 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ $0x00, R14 + ADCQ $0x00, BX + + // | a5 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ BX, R15 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a5 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, R15 + ADCQ BX, R8 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a5 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R15 + ADCQ DX, R8 + ADCQ BX, R9 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a5 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R8 + ADCQ DX, R9 + ADCQ $0x00, BX + + // | a5 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, BX + + // | + +/* */ + + // | + // | W + // | 0 (SP) | 1 8(SP) | 2 16(SP) | 3 R10 | 4 R11 | 5 R12 + // | 6 R13 | 7 R14 | 8 R15 | 9 R8 | 10 R9 | 11 BX + + + MOVQ (SP), CX + MOVQ 8(SP), DI + MOVQ 16(SP), SI + MOVQ BX, (SP) + MOVQ R9, 8(SP) + + // | + +/* montgomery reduction */ + + // | + +/* i0 */ + + // | + // | W + // | 0 CX | 1 DI | 2 SI | 3 R10 | 4 R11 | 5 R12 + // | 6 R13 | 7 R14 | 8 R15 | 9 R8 | 10 8(SP) | 11 (SP) + + + // | | u0 = w0 * inp + MOVQ CX, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | + +/* */ + + // | j0 + + // | w0 @ CX + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, CX + ADCQ DX, BX + + // | j1 + + // | w1 @ DI + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, DI + ADCQ $0x00, DX + ADDQ BX, DI + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + + // | w2 @ SI + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, SI + ADCQ $0x00, DX + ADDQ BX, SI + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + + // | w3 @ R10 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R10 + ADCQ $0x00, DX + ADDQ BX, R10 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + + // | w4 @ R11 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ BX, R11 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + + // | w5 @ R12 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + + // | w6 @ R13 + ADCQ DX, R13 + ADCQ $0x00, CX + + // | + +/* i1 */ + + // | + // | W + // | 0 - | 1 DI | 2 SI | 3 R10 | 4 R11 | 5 R12 + // | 6 R13 | 7 R14 | 8 R15 | 9 R8 | 10 8(SP) | 11 (SP) + + + // | | u1 = w1 * inp + MOVQ DI, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | + +/* */ + + // | j0 + + // | w1 @ DI + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, DI + ADCQ DX, BX + + // | j1 + + // | w2 @ SI + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, SI + ADCQ $0x00, DX + ADDQ BX, SI + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + + // | w3 @ R10 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R10 + ADCQ $0x00, DX + ADDQ BX, R10 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + + // | w4 @ R11 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ BX, R11 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + + // | w5 @ R12 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + + // | w6 @ R13 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ DX, CX + ADDQ BX, R13 + + // | w7 @ R14 + ADCQ CX, R14 + MOVQ $0x00, CX + ADCQ $0x00, CX + + // | + +/* i2 */ + + // | + // | W + // | 0 - | 1 - | 2 SI | 3 R10 | 4 R11 | 5 R12 + // | 6 R13 | 7 R14 | 8 R15 | 9 R8 | 10 8(SP) | 11 (SP) + + + // | | u2 = w2 * inp + MOVQ SI, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | + +/* */ + + // | j0 + + // | w2 @ SI + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, SI + ADCQ DX, BX + + // | j1 + + // | w3 @ R10 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, R10 + ADCQ $0x00, DX + ADDQ BX, R10 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + + // | w4 @ R11 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ BX, R11 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + + // | w5 @ R12 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + + // | w6 @ R13 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ BX, R13 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + + // | w7 @ R14 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R14 + ADCQ DX, CX + ADDQ BX, R14 + + // | w8 @ R15 + ADCQ CX, R15 + MOVQ $0x00, CX + ADCQ $0x00, CX + + // | + +/* i3 */ + + // | + // | W + // | 0 - | 1 - | 2 - | 3 R10 | 4 R11 | 5 R12 + // | 6 R13 | 7 R14 | 8 R15 | 9 R8 | 10 8(SP) | 11 (SP) + + + // | | u3 = w3 * inp + MOVQ R10, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | + +/* */ + + // | j0 + + // | w3 @ R10 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, R10 + ADCQ DX, BX + + // | j1 + + // | w4 @ R11 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ BX, R11 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + + // | w5 @ R12 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + + // | w6 @ R13 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ BX, R13 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + + // | w7 @ R14 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R14 + ADCQ $0x00, DX + ADDQ BX, R14 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + + // | w8 @ R15 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R15 + ADCQ DX, CX + ADDQ BX, R15 + + // | w9 @ R8 + ADCQ CX, R8 + MOVQ $0x00, CX + ADCQ $0x00, CX + + // | + +/* i4 */ + + // | + // | W + // | 0 - | 1 - | 2 - | 3 - | 4 R11 | 5 R12 + // | 6 R13 | 7 R14 | 8 R15 | 9 R8 | 10 8(SP) | 11 (SP) + + + // | | u4 = w4 * inp + MOVQ R11, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | + +/* */ + + // | j0 + + // | w4 @ R11 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ DX, BX + + // | j1 + + // | w5 @ R12 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + + // | w6 @ R13 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ BX, R13 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + + // | w7 @ R14 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R14 + ADCQ $0x00, DX + ADDQ BX, R14 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + + // | w8 @ R15 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R15 + ADCQ $0x00, DX + ADDQ BX, R15 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + + // | w9 @ R8 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R8 + ADCQ DX, CX + ADDQ BX, R8 + + // | move to idle register + MOVQ 8(SP), DI + + // | w10 @ DI + ADCQ CX, DI + MOVQ $0x00, CX + ADCQ $0x00, CX + + // | + +/* i5 */ + + // | + // | W + // | 0 - | 1 - | 2 - | 3 - | 4 - | 5 R12 + // | 6 R13 | 7 R14 | 8 R15 | 9 R8 | 10 DI | 11 (SP) + + + // | | u5 = w5 * inp + MOVQ R12, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | + +/* */ + + // | j0 + + // | w5 @ R12 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ DX, BX + + // | j1 + + // | w6 @ R13 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ BX, R13 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + + // | w7 @ R14 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R14 + ADCQ $0x00, DX + ADDQ BX, R14 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + + // | w8 @ R15 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R15 + ADCQ $0x00, DX + ADDQ BX, R15 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + + // | w9 @ R8 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R8 + ADCQ $0x00, DX + ADDQ BX, R8 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + + // | w10 @ DI + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, DI + ADCQ DX, CX + ADDQ BX, DI + + // | w11 @ CX + ADCQ (SP), CX + + // | + // | W montgomerry reduction ends + // | 0 - | 1 - | 2 - | 3 - | 4 - | 5 - + // | 6 R13 | 7 R14 | 8 R15 | 9 R8 | 10 DI | 11 CX + + + // | + + +/* modular reduction */ + + MOVQ R13, R10 + SUBQ ·modulus+0(SB), R10 + MOVQ R14, R11 + SBBQ ·modulus+8(SB), R11 + MOVQ R15, R12 + SBBQ ·modulus+16(SB), R12 + MOVQ R8, AX + SBBQ ·modulus+24(SB), AX + MOVQ DI, BX + SBBQ ·modulus+32(SB), BX + MOVQ CX, R9 + SBBQ ·modulus+40(SB), R9 + // | + +/* out */ + + MOVQ c+0(FP), SI + CMOVQCC R10, R13 + MOVQ R13, (SI) + CMOVQCC R11, R14 + MOVQ R14, 8(SI) + CMOVQCC R12, R15 + MOVQ R15, 16(SI) + CMOVQCC AX, R8 + MOVQ R8, 24(SI) + CMOVQCC BX, DI + MOVQ DI, 32(SI) + CMOVQCC R9, CX + MOVQ CX, 40(SI) + RET + + // | + +/* end */ + + +// multiplication +// c = a * b % p +TEXT ·mulADX(SB), NOSPLIT, $16-24 + // | + +/* inputs */ + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + XORQ AX, AX + + // | + +/* i0 */ + + // | a0 @ DX + MOVQ (DI), DX + + // | a0 * b0 + MULXQ (SI), AX, CX + MOVQ AX, (SP) + + // | a0 * b1 + MULXQ 8(SI), AX, R8 + ADCXQ AX, CX + + // | a0 * b2 + MULXQ 16(SI), AX, R9 + ADCXQ AX, R8 + + // | a0 * b3 + MULXQ 24(SI), AX, R10 + ADCXQ AX, R9 + + // | a0 * b4 + MULXQ 32(SI), AX, R11 + ADCXQ AX, R10 + + // | a0 * b5 + MULXQ 40(SI), AX, R12 + ADCXQ AX, R11 + ADCQ $0x00, R12 + + // | + +/* i1 */ + + // | a1 @ DX + MOVQ 8(DI), DX + XORQ R13, R13 + + // | a1 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, CX + ADCXQ BX, R8 + MOVQ CX, 8(SP) + + // | a1 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R8 + ADCXQ BX, R9 + + // | a1 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | a1 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a1 * b4 + MULXQ 32(SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a1 * b5 + MULXQ 40(SI), AX, BX + ADOXQ AX, R12 + ADOXQ R13, R13 + ADCXQ BX, R13 + + // | + +/* i2 */ + + // | a2 @ DX + MOVQ 16(DI), DX + XORQ R14, R14 + + // | a2 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R8 + ADCXQ BX, R9 + + // | a2 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | a2 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a2 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a2 * b4 + MULXQ 32(SI), AX, BX + ADOXQ AX, R12 + ADCXQ BX, R13 + + // | a2 * b5 + MULXQ 40(SI), AX, BX + ADOXQ AX, R13 + ADOXQ R14, R14 + ADCXQ BX, R14 + + // | + +/* i3 */ + + // | a3 @ DX + MOVQ 24(DI), DX + XORQ R15, R15 + + // | a3 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | a3 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a3 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a3 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R12 + ADCXQ BX, R13 + + // | a3 * b4 + MULXQ 32(SI), AX, BX + ADOXQ AX, R13 + ADCXQ BX, R14 + + // | a3 * b5 + MULXQ 40(SI), AX, BX + ADOXQ AX, R14 + ADOXQ R15, R15 + ADCXQ BX, R15 + + // | + +/* i4 */ + + // | a4 @ DX + MOVQ 32(DI), DX + XORQ CX, CX + + // | a4 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a4 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a4 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R12 + ADCXQ BX, R13 + + // | a4 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R13 + ADCXQ BX, R14 + + // | a4 * b4 + MULXQ 32(SI), AX, BX + ADOXQ AX, R14 + ADCXQ BX, R15 + + // | a4 * b5 + MULXQ 40(SI), AX, BX + ADOXQ AX, R15 + ADOXQ CX, CX + ADCXQ BX, CX + + // | + +/* i5 */ + + // | a5 @ DX + MOVQ 40(DI), DX + XORQ DI, DI + + // | a5 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a5 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R12 + ADCXQ BX, R13 + + // | a5 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R13 + ADCXQ BX, R14 + + // | a5 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R14 + ADCXQ BX, R15 + + // | a5 * b4 + MULXQ 32(SI), AX, BX + ADOXQ AX, R15 + ADCXQ BX, CX + + // | a5 * b5 + MULXQ 40(SI), AX, BX + ADOXQ AX, CX + ADOXQ BX, DI + ADCQ $0x00, DI + + // | + +/* */ + + // | + // | W + // | 0 (SP) | 1 8(SP) | 2 R8 | 3 R9 | 4 R10 | 5 R11 + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 DI + + + MOVQ (SP), BX + MOVQ 8(SP), SI + MOVQ DI, (SP) + + // | + // | W ready to mont + // | 0 BX | 1 SI | 2 R8 | 3 R9 | 4 R10 | 5 R11 + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 (SP) + + + // | + +/* montgomery reduction */ + + // | clear flags + XORQ AX, AX + + // | + +/* i0 */ + + // | + // | W + // | 0 BX | 1 SI | 2 R8 | 3 R9 | 4 R10 | 5 R11 + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 (SP) + + + // | | u0 = w0 * inp + MOVQ BX, DX + MULXQ ·inp+0(SB), DX, DI + + // | + +/* */ + + // | j0 + + // | w0 @ BX + MULXQ ·modulus+0(SB), AX, DI + ADOXQ AX, BX + ADCXQ DI, SI + + // | j1 + + // | w1 @ SI + MULXQ ·modulus+8(SB), AX, DI + ADOXQ AX, SI + ADCXQ DI, R8 + + // | j2 + + // | w2 @ R8 + MULXQ ·modulus+16(SB), AX, DI + ADOXQ AX, R8 + ADCXQ DI, R9 + + // | j3 + + // | w3 @ R9 + MULXQ ·modulus+24(SB), AX, DI + ADOXQ AX, R9 + ADCXQ DI, R10 + + // | j4 + + // | w4 @ R10 + MULXQ ·modulus+32(SB), AX, DI + ADOXQ AX, R10 + ADCXQ DI, R11 + + // | j5 + + // | w5 @ R11 + MULXQ ·modulus+40(SB), AX, DI + ADOXQ AX, R11 + ADCXQ DI, R12 + ADOXQ BX, R12 + ADCXQ BX, BX + MOVQ $0x00, AX + ADOXQ AX, BX + + // | clear flags + XORQ AX, AX + + // | + +/* i1 */ + + // | + // | W + // | 0 - | 1 SI | 2 R8 | 3 R9 | 4 R10 | 5 R11 + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 (SP) + + + // | | u1 = w1 * inp + MOVQ SI, DX + MULXQ ·inp+0(SB), DX, DI + + // | + +/* */ + + // | j0 + + // | w1 @ SI + MULXQ ·modulus+0(SB), AX, DI + ADOXQ AX, SI + ADCXQ DI, R8 + + // | j1 + + // | w2 @ R8 + MULXQ ·modulus+8(SB), AX, DI + ADOXQ AX, R8 + ADCXQ DI, R9 + + // | j2 + + // | w3 @ R9 + MULXQ ·modulus+16(SB), AX, DI + ADOXQ AX, R9 + ADCXQ DI, R10 + + // | j3 + + // | w4 @ R10 + MULXQ ·modulus+24(SB), AX, DI + ADOXQ AX, R10 + ADCXQ DI, R11 + + // | j4 + + // | w5 @ R11 + MULXQ ·modulus+32(SB), AX, DI + ADOXQ AX, R11 + ADCXQ DI, R12 + + // | j5 + + // | w6 @ R12 + MULXQ ·modulus+40(SB), AX, DI + ADOXQ AX, R12 + ADCXQ DI, R13 + ADOXQ BX, R13 + ADCXQ SI, SI + MOVQ $0x00, AX + ADOXQ AX, SI + + // | clear flags + XORQ AX, AX + + // | + +/* i2 */ + + // | + // | W + // | 0 - | 1 - | 2 R8 | 3 R9 | 4 R10 | 5 R11 + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 (SP) + + + // | | u2 = w2 * inp + MOVQ R8, DX + MULXQ ·inp+0(SB), DX, DI + + // | + +/* */ + + // | j0 + + // | w2 @ R8 + MULXQ ·modulus+0(SB), AX, DI + ADOXQ AX, R8 + ADCXQ DI, R9 + + // | j1 + + // | w3 @ R9 + MULXQ ·modulus+8(SB), AX, DI + ADOXQ AX, R9 + ADCXQ DI, R10 + + // | j2 + + // | w4 @ R10 + MULXQ ·modulus+16(SB), AX, DI + ADOXQ AX, R10 + ADCXQ DI, R11 + + // | j3 + + // | w5 @ R11 + MULXQ ·modulus+24(SB), AX, DI + ADOXQ AX, R11 + ADCXQ DI, R12 + + // | j4 + + // | w6 @ R12 + MULXQ ·modulus+32(SB), AX, DI + ADOXQ AX, R12 + ADCXQ DI, R13 + + // | j5 + + // | w7 @ R13 + MULXQ ·modulus+40(SB), AX, DI + ADOXQ AX, R13 + ADCXQ DI, R14 + ADOXQ SI, R14 + ADCXQ R8, R8 + MOVQ $0x00, AX + ADOXQ AX, R8 + + // | clear flags + XORQ AX, AX + + // | + +/* i3 */ + + // | + // | W + // | 0 - | 1 - | 2 - | 3 R9 | 4 R10 | 5 R11 + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 (SP) + + + // | | u3 = w3 * inp + MOVQ R9, DX + MULXQ ·inp+0(SB), DX, DI + + // | + +/* */ + + // | j0 + + // | w3 @ R9 + MULXQ ·modulus+0(SB), AX, DI + ADOXQ AX, R9 + ADCXQ DI, R10 + + // | j1 + + // | w4 @ R10 + MULXQ ·modulus+8(SB), AX, DI + ADOXQ AX, R10 + ADCXQ DI, R11 + + // | j2 + + // | w5 @ R11 + MULXQ ·modulus+16(SB), AX, DI + ADOXQ AX, R11 + ADCXQ DI, R12 + + // | j3 + + // | w6 @ R12 + MULXQ ·modulus+24(SB), AX, DI + ADOXQ AX, R12 + ADCXQ DI, R13 + + // | j4 + + // | w7 @ R13 + MULXQ ·modulus+32(SB), AX, DI + ADOXQ AX, R13 + ADCXQ DI, R14 + + // | j5 + + // | w8 @ R14 + MULXQ ·modulus+40(SB), AX, DI + ADOXQ AX, R14 + ADCXQ DI, R15 + ADOXQ R8, R15 + ADCXQ R9, R9 + MOVQ $0x00, AX + ADOXQ AX, R9 + + // | clear flags + XORQ AX, AX + + // | + +/* i4 */ + + // | + // | W + // | 0 - | 1 - | 2 - | 3 - | 4 R10 | 5 R11 + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 (SP) + + + // | | u4 = w4 * inp + MOVQ R10, DX + MULXQ ·inp+0(SB), DX, DI + + // | + +/* */ + + // | j0 + + // | w4 @ R10 + MULXQ ·modulus+0(SB), AX, DI + ADOXQ AX, R10 + ADCXQ DI, R11 + + // | j1 + + // | w5 @ R11 + MULXQ ·modulus+8(SB), AX, DI + ADOXQ AX, R11 + ADCXQ DI, R12 + + // | j2 + + // | w6 @ R12 + MULXQ ·modulus+16(SB), AX, DI + ADOXQ AX, R12 + ADCXQ DI, R13 + + // | j3 + + // | w7 @ R13 + MULXQ ·modulus+24(SB), AX, DI + ADOXQ AX, R13 + ADCXQ DI, R14 + + // | j4 + + // | w8 @ R14 + MULXQ ·modulus+32(SB), AX, DI + ADOXQ AX, R14 + ADCXQ DI, R15 + + // | j5 + + // | w9 @ R15 + MULXQ ·modulus+40(SB), AX, DI + ADOXQ AX, R15 + ADCXQ DI, CX + ADOXQ R9, CX + ADCXQ R10, R10 + MOVQ $0x00, AX + ADOXQ AX, R10 + + // | clear flags + XORQ AX, AX + + // | + +/* i5 */ + + // | + // | W + // | 0 - | 1 - | 2 - | 3 - | 4 - | 5 R11 + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 (SP) + + + // | | u5 = w5 * inp + MOVQ R11, DX + MULXQ ·inp+0(SB), DX, DI + + // | + +/* */ + + // | j0 + + // | w5 @ R11 + MULXQ ·modulus+0(SB), AX, DI + ADOXQ AX, R11 + ADCXQ DI, R12 + + // | j1 + + // | w6 @ R12 + MULXQ ·modulus+8(SB), AX, DI + ADOXQ AX, R12 + ADCXQ DI, R13 + + // | j2 + + // | w7 @ R13 + MULXQ ·modulus+16(SB), AX, DI + ADOXQ AX, R13 + ADCXQ DI, R14 + + // | j3 + + // | w8 @ R14 + MULXQ ·modulus+24(SB), AX, DI + ADOXQ AX, R14 + ADCXQ DI, R15 + + // | j4 + + // | w9 @ R15 + MULXQ ·modulus+32(SB), AX, DI + ADOXQ AX, R15 + ADCXQ DI, CX + + // | j5 + + // | w10 @ CX + MULXQ ·modulus+40(SB), AX, DI + ADOXQ AX, CX + + // | w11 @ (SP) + // | move to an idle register + MOVQ (SP), BX + ADCXQ DI, BX + ADOXQ R10, BX + + // | + // | W montgomery reduction ends + // | 0 - | 1 - | 2 - | 3 - | 4 - | 5 - + // | 6 R12 | 7 R13 | 8 R14 | 9 R15 | 10 CX | 11 BX + + + // | + +/* modular reduction */ + + MOVQ R12, AX + SUBQ ·modulus+0(SB), AX + MOVQ R13, DI + SBBQ ·modulus+8(SB), DI + MOVQ R14, SI + SBBQ ·modulus+16(SB), SI + MOVQ R15, R8 + SBBQ ·modulus+24(SB), R8 + MOVQ CX, R9 + SBBQ ·modulus+32(SB), R9 + MOVQ BX, R10 + SBBQ ·modulus+40(SB), R10 + + // | + +/* out */ + + MOVQ c+0(FP), R11 + CMOVQCC AX, R12 + MOVQ R12, (R11) + CMOVQCC DI, R13 + MOVQ R13, 8(R11) + CMOVQCC SI, R14 + MOVQ R14, 16(R11) + CMOVQCC R8, R15 + MOVQ R15, 24(R11) + CMOVQCC R9, CX + MOVQ CX, 32(R11) + CMOVQCC R10, BX + MOVQ BX, 40(R11) + RET + + // | + +/* end */ + + +// c = a * b +TEXT ·wmulADX(SB), NOSPLIT, $0-24 + // | + +/* inputs */ + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ c+0(FP), R8 + XORQ BX, BX + +/* i0 */ + + MOVQ (DI), DX + + // | a0 * b0 + MULXQ (SI), AX, CX + MOVQ AX, 0(R8) + + // | a0 * b1 + MULXQ 8(SI), AX, BP + ADCXQ AX, CX + + // | a0 * b2 + MULXQ 16(SI), AX, R9 + ADCXQ AX, BP + + // | a0 * b3 + MULXQ 24(SI), AX, R10 + ADCXQ AX, R9 + + // | a0 * b4 + MULXQ 32(SI), AX, R11 + ADCXQ AX, R10 + + // | a0 * b5 + MULXQ 40(SI), AX, R12 + ADCXQ AX, R11 + ADCXQ BX, R12 + +/* i1 */ + + MOVQ 8(DI), DX + + // | a1 * b0 + MULXQ (SI), AX, R13 + ADOXQ AX, CX + ADCXQ R13, BP + MOVQ CX, 8(R8) + + // | a1 * b1 + MULXQ 8(SI), AX, R13 + ADOXQ AX, BP + ADCXQ R13, R9 + + // | a1 * b2 + MULXQ 16(SI), AX, R13 + ADOXQ AX, R9 + ADCXQ R13, R10 + + // | a1 * b3 + MULXQ 24(SI), AX, R13 + ADOXQ AX, R10 + ADCXQ R13, R11 + + // | a1 * b4 + MULXQ 32(SI), AX, R13 + ADOXQ AX, R11 + ADCXQ R13, R12 + + // | a1 * b5 + MULXQ 40(SI), AX, R13 + ADOXQ AX, R12 + ADOXQ BX, R13 + ADCXQ BX, R13 + +/* i2 */ + + MOVQ 16(DI), DX + + // | a2 * b0 + MULXQ (SI), AX, R14 + ADOXQ AX, BP + ADCXQ R14, R9 + MOVQ BP, 16(R8) + + // | a2 * b1 + MULXQ 8(SI), AX, R14 + ADOXQ AX, R9 + ADCXQ R14, R10 + + // | a2 * b2 + MULXQ 16(SI), AX, R14 + ADOXQ AX, R10 + ADCXQ R14, R11 + + // | a2 * b3 + MULXQ 24(SI), AX, R14 + ADOXQ AX, R11 + ADCXQ R14, R12 + + // | a2 * b4 + MULXQ 32(SI), AX, R14 + ADOXQ AX, R12 + ADCXQ R14, R13 + + // | a2 * b5 + MULXQ 40(SI), AX, R14 + ADOXQ AX, R13 + ADOXQ BX, R14 + ADCXQ BX, R14 + +/* i3 */ + + MOVQ 24(DI), DX + + // | a3 * b0 + MULXQ (SI), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + MOVQ R9, 24(R8) + + // | a3 * b1 + MULXQ 8(SI), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | a3 * b2 + MULXQ 16(SI), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | a3 * b3 + MULXQ 24(SI), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | a3 * b4 + MULXQ 32(SI), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | a3 * b5 + MULXQ 40(SI), AX, R15 + ADOXQ AX, R14 + ADOXQ BX, R15 + ADCXQ BX, R15 + +/* i4 */ + + MOVQ 32(DI), DX + + // | a4 * b0 + MULXQ (SI), AX, CX + ADOXQ AX, R10 + ADCXQ CX, R11 + MOVQ R10, 32(R8) + + // | a4 * b1 + MULXQ 8(SI), AX, CX + ADOXQ AX, R11 + ADCXQ CX, R12 + + // | a4 * b2 + MULXQ 16(SI), AX, CX + ADOXQ AX, R12 + ADCXQ CX, R13 + + // | a4 * b3 + MULXQ 24(SI), AX, CX + ADOXQ AX, R13 + ADCXQ CX, R14 + + // | a4 * b4 + MULXQ 32(SI), AX, CX + ADOXQ AX, R14 + ADCXQ CX, R15 + + // | a4 * b5 + MULXQ 40(SI), AX, CX + ADOXQ AX, R15 + ADOXQ BX, CX + ADCXQ BX, CX + + +/* i5 */ + + MOVQ 40(DI), DX + + // | a5 * b0 + MULXQ (SI), AX, DI + ADOXQ AX, R11 + ADCXQ DI, R12 + MOVQ R11, 40(R8) + + // | a5 * b1 + MULXQ 8(SI), AX, DI + ADOXQ AX, R12 + ADCXQ DI, R13 + + // | a5 * b2 + MULXQ 16(SI), AX, DI + ADOXQ AX, R13 + ADCXQ DI, R14 + + // | a5 * b3 + MULXQ 24(SI), AX, DI + ADOXQ AX, R14 + ADCXQ DI, R15 + + // | a5 * b4 + MULXQ 32(SI), AX, DI + ADOXQ AX, R15 + ADCXQ DI, CX + + // | a5 * b5 + MULXQ 40(SI), AX, DI + ADOXQ AX, CX + ADOXQ BX, DI + ADCXQ BX, DI + + + MOVQ R12, 48(R8) + MOVQ R13, 56(R8) + MOVQ R14, 64(R8) + MOVQ R15, 72(R8) + MOVQ CX, 80(R8) + MOVQ DI, 88(R8) + RET + + +TEXT ·wmulNoADX(SB), NOSPLIT, $0-24 + + MOVQ c+0(FP), BP + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ $0x00, R9 + MOVQ $0x00, R10 + MOVQ $0x00, R11 + MOVQ $0x00, R12 + MOVQ $0x00, R13 + MOVQ $0x00, R14 + MOVQ $0x00, R15 + + // | + +/* i0 */ + + MOVQ (DI), CX + + // | a0 * b0 + MOVQ (SI), AX + MULQ CX + MOVQ AX, 0(BP) + MOVQ DX, R8 + + // | a0 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R8 + ADCQ DX, R9 + + // | a0 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + + // | a0 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + + // | a0 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + + // | a0 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + +/* i1 */ + + MOVQ 8(DI), CX + MOVQ $0x00, BX + + // | a1 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R8 + ADCQ DX, R9 + ADCQ $0x00, R10 + ADCQ $0x00, BX + MOVQ R8, 8(BP) + MOVQ $0x00, R8 + + // | a1 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + ADCQ BX, R11 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a1 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ BX, R12 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a1 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a1 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + + // | a1 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + +/* i2 */ + + MOVQ 16(DI), CX + MOVQ $0x00, BX + + // | a2 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + ADCQ $0x00, R11 + ADCQ $0x00, BX + MOVQ R9, 16(BP) + MOVQ $0x00, R9 + + // | a2 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ BX, R12 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a2 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a2 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a2 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ BX, R15 + + // | a2 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, R15 + +/* i3 */ + + MOVQ 24(DI), CX + MOVQ $0x00, BX + + // | a3 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ $0x00, R12 + ADCQ $0x00, BX + MOVQ R10, 24(BP) + + // | a3 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a3 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a3 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ BX, R15 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a3 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, R15 + ADCQ BX, R8 + + // | a3 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R15 + ADCQ DX, R8 + +/* i4 */ + + MOVQ 32(DI), CX + MOVQ $0x00, BX + + // | a4 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ $0x00, R13 + ADCQ $0x00, BX + MOVQ R11, 32(BP) + + // | a4 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a4 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ BX, R15 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a4 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, R15 + ADCQ BX, R8 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a4 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R15 + ADCQ DX, R8 + ADCQ BX, R9 + + // | a4 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R8 + ADCQ DX, R9 + +/* i5 */ + + MOVQ 40(DI), CX + MOVQ $0x00, BX + + // | a5 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ $0x00, R14 + ADCQ $0x00, BX + MOVQ R12, 40(BP) + + // | a5 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ BX, R15 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a5 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, R15 + ADCQ BX, R8 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a5 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R15 + ADCQ DX, R8 + ADCQ BX, R9 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a5 * b4 + MOVQ 32(SI), AX + MULQ CX + ADDQ AX, R8 + ADCQ DX, R9 + ADCQ $0x00, BX + + // | a5 * b5 + MOVQ 40(SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, BX + + MOVQ R13, 48(BP) + MOVQ R14, 56(BP) + MOVQ R15, 64(BP) + MOVQ R8, 72(BP) + MOVQ R9, 80(BP) + MOVQ BX, 88(BP) + + RET + + +TEXT ·lwadd(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), DX + + MOVQ c+0(FP), SI + MOVQ R8, (SI) + MOVQ R9, 8(SI) + MOVQ R10, 16(SI) + MOVQ R11, 24(SI) + MOVQ R12, 32(SI) + MOVQ R13, 40(SI) + MOVQ R14, 48(SI) + MOVQ R15, 56(SI) + MOVQ AX, 64(SI) + MOVQ BX, 72(SI) + MOVQ CX, 80(SI) + MOVQ DX, 88(SI) + RET + + +TEXT ·lwaddAssign(SB), NOSPLIT, $0-16 + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), DX + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + RET + + +// double-precision addition w/ upper bound check + // if c > (2^N)p , + // then correct by c = c - (2^N)p + // c = a + b + TEXT ·wadd(SB), NOSPLIT, $0-24 + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + ADCQ 64(SI), AX + ADCQ 72(SI), BX + ADCQ 80(SI), CX + ADCQ 88(SI), DX + + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, SI + SUBQ SI, R8 + MOVQ $0x1eabfffeb153ffff, SI + SBBQ SI, R9 + MOVQ $0x6730d2a0f6b0f624, SI + SBBQ SI, R10 + MOVQ $0x64774b84f38512bf, SI + SBBQ SI, R11 + MOVQ $0x4b1ba7b6434bacd7, SI + SBBQ SI, R12 + MOVQ $0x1a0111ea397fe69a, SI + SBBQ SI, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + RET + + +TEXT ·lwsub(SB), NOSPLIT, $0-24 + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + SBBQ 48(SI), R14 + SBBQ 56(SI), R15 + SBBQ 64(SI), AX + SBBQ 72(SI), BX + SBBQ 80(SI), CX + SBBQ 88(SI), DX + + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + RET + + +TEXT ·lwsubAssign(SB), NOSPLIT, $0-16 + + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + SBBQ 48(SI), R14 + SBBQ 56(SI), R15 + SBBQ 64(SI), AX + SBBQ 72(SI), BX + SBBQ 80(SI), CX + SBBQ 88(SI), DX + + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + RET + + +// double-precision subtraction. +// [AKLGL] Option2 +// https://eprint.iacr.org/2010/526 +// c = (a - b) +// if c is negative, +// then correct by c = c + (2^N)p +TEXT ·wsub(SB), NOSPLIT, $0-24 + + // | + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + SUBQ (SI), R8 + SBBQ 8(SI), R9 + SBBQ 16(SI), R10 + SBBQ 24(SI), R11 + SBBQ 32(SI), R12 + SBBQ 40(SI), R13 + SBBQ 48(SI), R14 + SBBQ 56(SI), R15 + SBBQ 64(SI), AX + SBBQ 72(SI), BX + SBBQ 80(SI), CX + SBBQ 88(SI), DX + + MOVQ c+0(FP), DI + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + + MOVQ $0, SI + MOVQ $0xb9feffffffffaaab, R8 + MOVQ $0x1eabfffeb153ffff, R9 + MOVQ $0x6730d2a0f6b0f624, R10 + MOVQ $0x64774b84f38512bf, R11 + MOVQ $0x4b1ba7b6434bacd7, R12 + MOVQ $0x1a0111ea397fe69a, R13 + CMOVQCC SI, R8 + CMOVQCC SI, R9 + CMOVQCC SI, R10 + CMOVQCC SI, R11 + CMOVQCC SI, R12 + CMOVQCC SI, R13 + ADDQ R8, R14 + ADCQ R9, R15 + ADCQ R10, AX + ADCQ R11, BX + ADCQ R12, CX + ADCQ R13, DX + + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + MOVQ AX, 64(DI) + MOVQ BX, 72(DI) + MOVQ CX, 80(DI) + MOVQ DX, 88(DI) + RET + + + // double-precision doubling w/ upper bound check + // if c > (2^N)p , + // then correct by c = c - (2^N)p + // c = 2 * a + TEXT ·wdouble(SB), NOSPLIT, $0-16 + // | + MOVQ a+8(FP), DI + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + // | + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + ADCQ R14, R14 + ADCQ R15, R15 + ADCQ AX, AX + ADCQ BX, BX + ADCQ CX, CX + ADCQ DX, DX + + // | + MOVQ c+0(FP), SI + MOVQ R8, (SI) + MOVQ R9, 8(SI) + MOVQ R10, 16(SI) + MOVQ R11, 24(SI) + MOVQ R12, 32(SI) + MOVQ R13, 40(SI) + + // | + MOVQ R14, R8 + MOVQ R15, R9 + MOVQ AX, R10 + MOVQ BX, R11 + MOVQ CX, R12 + MOVQ DX, R13 + MOVQ $0xb9feffffffffaaab, DI + SUBQ DI, R8 + MOVQ $0x1eabfffeb153ffff, DI + SBBQ DI, R9 + MOVQ $0x6730d2a0f6b0f624, DI + SBBQ DI, R10 + MOVQ $0x64774b84f38512bf, DI + SBBQ DI, R11 + MOVQ $0x4b1ba7b6434bacd7, DI + SBBQ DI, R12 + MOVQ $0x1a0111ea397fe69a, DI + SBBQ DI, R13 + CMOVQCC R8, R14 + CMOVQCC R9, R15 + CMOVQCC R10, AX + CMOVQCC R11, BX + CMOVQCC R12, CX + CMOVQCC R13, DX + + // | + MOVQ R14, 48(SI) + MOVQ R15, 56(SI) + MOVQ AX, 64(SI) + MOVQ BX, 72(SI) + MOVQ CX, 80(SI) + MOVQ DX, 88(SI) + RET + + + TEXT ·lwdouble(SB), NOSPLIT, $0-16 + // | + MOVQ a+8(FP), DI + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + MOVQ 64(DI), AX + MOVQ 72(DI), BX + MOVQ 80(DI), CX + MOVQ 88(DI), DX + + // | + ADDQ R8, R8 + ADCQ R9, R9 + ADCQ R10, R10 + ADCQ R11, R11 + ADCQ R12, R12 + ADCQ R13, R13 + ADCQ R14, R14 + ADCQ R15, R15 + ADCQ AX, AX + ADCQ BX, BX + ADCQ CX, CX + ADCQ DX, DX + + // | + MOVQ c+0(FP), SI + MOVQ R8, (SI) + MOVQ R9, 8(SI) + MOVQ R10, 16(SI) + MOVQ R11, 24(SI) + MOVQ R12, 32(SI) + MOVQ R13, 40(SI) + MOVQ R14, 48(SI) + MOVQ R15, 56(SI) + MOVQ AX, 64(SI) + MOVQ BX, 72(SI) + MOVQ CX, 80(SI) + MOVQ DX, 88(SI) + RET + + +TEXT ·montRedADX(SB), NOSPLIT, $0-16 + + MOVQ w+8(FP), DI + MOVQ 0(DI), BX + MOVQ 8(DI), SI + MOVQ 16(DI), R8 + MOVQ 24(DI), R9 + MOVQ 32(DI), R10 + MOVQ 40(DI), R11 + MOVQ 48(DI), R12 + MOVQ 56(DI), R13 + MOVQ 64(DI), R14 + + XORQ CX, CX + + MOVQ BX, DX + MULXQ ·inp+0(SB), DX, R15 + + // | j0 + MULXQ ·modulus+0(SB), AX, R15 + ADOXQ AX, BX + ADCXQ R15, SI + + // | j1 + MULXQ ·modulus+8(SB), AX, R15 + ADOXQ AX, SI + ADCXQ R15, R8 + + // | j2 + MULXQ ·modulus+16(SB), AX, R15 + ADOXQ AX, R8 + ADCXQ R15, R9 + + // | j3 + MULXQ ·modulus+24(SB), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + + // | j4 + MULXQ ·modulus+32(SB), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | j5 + MULXQ ·modulus+40(SB), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + ADOXQ CX, R12 + ADCXQ CX, BX + ADOXQ CX, BX + +/* i1 */ + + MOVQ SI, DX + MULXQ ·inp+0(SB), DX, R15 + + // | j0 + MULXQ ·modulus+0(SB), AX, R15 + ADOXQ AX, SI + ADCXQ R15, R8 + + // | j1 + MULXQ ·modulus+8(SB), AX, R15 + ADOXQ AX, R8 + ADCXQ R15, R9 + + // | j2 + MULXQ ·modulus+16(SB), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + + // | j3 + MULXQ ·modulus+24(SB), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | j4 + MULXQ ·modulus+32(SB), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | j5 + MULXQ ·modulus+40(SB), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + ADOXQ BX, R13 + ADCXQ CX, SI + ADOXQ CX, SI + + MOVQ 72(DI), BX + +/* i2 */ + + MOVQ R8, DX + MULXQ ·inp+0(SB), DX, R15 + + + // | j0 + MULXQ ·modulus+0(SB), AX, R15 + ADOXQ AX, R8 + ADCXQ R15, R9 + + // | j1 + MULXQ ·modulus+8(SB), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + + // | j2 + MULXQ ·modulus+16(SB), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | j3 + MULXQ ·modulus+24(SB), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | j4 + MULXQ ·modulus+32(SB), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | j5 + MULXQ ·modulus+40(SB), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + ADOXQ SI, R14 + ADCXQ CX, R8 + ADOXQ CX, R8 + + MOVQ 80(DI), SI + +/* i3 */ + + MOVQ R9, DX + MULXQ ·inp+0(SB), DX, R15 + + // | j0 + MULXQ ·modulus+0(SB), AX, R15 + ADOXQ AX, R9 + ADCXQ R15, R10 + + // | j1 + MULXQ ·modulus+8(SB), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | j2 + MULXQ ·modulus+16(SB), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | j3 + MULXQ ·modulus+24(SB), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | j4 + MULXQ ·modulus+32(SB), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | j5 + MULXQ ·modulus+40(SB), AX, R15 + ADOXQ AX, R14 + ADCXQ R15, BX + ADOXQ R8, BX + ADCXQ CX, R9 + ADOXQ CX, R9 + + MOVQ 88(DI), R8 + + +/* i4 */ + + MOVQ R10, DX + MULXQ ·inp+0(SB), DX, R15 + + // | j0 + MULXQ ·modulus+0(SB), AX, R15 + ADOXQ AX, R10 + ADCXQ R15, R11 + + // | j1 + MULXQ ·modulus+8(SB), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | j2 + MULXQ ·modulus+16(SB), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | j3 + MULXQ ·modulus+24(SB), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | j4 + MULXQ ·modulus+32(SB), AX, R15 + ADOXQ AX, R14 + ADCXQ R15, BX + + // | j5 + MULXQ ·modulus+40(SB), AX, R15 + ADOXQ AX, BX + ADCXQ R15, SI + ADOXQ R9, SI + ADCXQ CX, R10 + ADOXQ CX, R10 + + // | + +/* i5 */ + + MOVQ R11, DX + MULXQ ·inp+0(SB), DX, R15 + + // | j0 + MULXQ ·modulus+0(SB), AX, R15 + ADOXQ AX, R11 + ADCXQ R15, R12 + + // | j1 + MULXQ ·modulus+8(SB), AX, R15 + ADOXQ AX, R12 + ADCXQ R15, R13 + + // | j2 + MULXQ ·modulus+16(SB), AX, R15 + ADOXQ AX, R13 + ADCXQ R15, R14 + + // | j3 + MULXQ ·modulus+24(SB), AX, R15 + ADOXQ AX, R14 + ADCXQ R15, BX + + // | j4 + MULXQ ·modulus+32(SB), AX, R15 + ADOXQ AX, BX + ADCXQ R15, SI + + // | j5 + MULXQ ·modulus+40(SB), AX, R15 + ADOXQ AX, SI + ADCXQ R15, R8 + ADOXQ R10, R8 + +/* modular reduction */ + + MOVQ R12, AX + MOVQ R13, DI + MOVQ R14, CX + MOVQ BX, DX + MOVQ SI, R9 + MOVQ R8, R10 + + SUBQ ·modulus+0(SB), AX + SBBQ ·modulus+8(SB), DI + SBBQ ·modulus+16(SB), CX + SBBQ ·modulus+24(SB), DX + SBBQ ·modulus+32(SB), R9 + SBBQ ·modulus+40(SB), R10 + + CMOVQCC AX, R12 + CMOVQCC DI, R13 + CMOVQCC CX, R14 + CMOVQCC DX, BX + CMOVQCC R9, SI + CMOVQCC R10, R8 + + MOVQ a+0(FP), R11 + MOVQ R12, (R11) + MOVQ R13, 8(R11) + MOVQ R14, 16(R11) + MOVQ BX, 24(R11) + MOVQ SI, 32(R11) + MOVQ R8, 40(R11) + RET + +/* end */ + + +TEXT ·montRedNoADX(SB), NOSPLIT, $0-16 + + MOVQ w+8(FP), BP + MOVQ 0(BP), CX + MOVQ 8(BP), DI + MOVQ 16(BP), SI + MOVQ 24(BP), R10 + MOVQ 32(BP), R11 + MOVQ 40(BP), R12 + MOVQ 48(BP), R13 + MOVQ 56(BP), R14 + MOVQ 64(BP), R15 + MOVQ 72(BP), R8 + +/* i0 */ + + MOVQ CX, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | j0 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, CX + ADCQ DX, BX + + // | j1 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, DI + ADCQ $0x00, DX + ADDQ BX, DI + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, SI + ADCQ $0x00, DX + ADDQ BX, SI + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R10 + ADCQ $0x00, DX + ADDQ BX, R10 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ BX, R11 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + + ADCQ DX, R13 + ADCQ $0x00, CX + + +/* i1 */ + + MOVQ DI, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + + // | j0 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, DI + ADCQ DX, BX + + // | j1 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, SI + ADCQ $0x00, DX + ADDQ BX, SI + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R10 + ADCQ $0x00, DX + ADDQ BX, R10 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ BX, R11 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ DX, CX + ADDQ BX, R13 + + ADCQ CX, R14 + MOVQ $0x00, CX + ADCQ $0x00, CX + + +/* i2 */ + + MOVQ SI, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + +/* */ + + // | j0 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, SI + ADCQ DX, BX + + // | j1 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, R10 + ADCQ $0x00, DX + ADDQ BX, R10 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ BX, R11 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ BX, R13 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R14 + ADCQ DX, CX + ADDQ BX, R14 + + ADCQ CX, R15 + MOVQ $0x00, CX + ADCQ $0x00, CX + +/* i3 */ + + MOVQ R10, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | j0 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, R10 + ADCQ DX, BX + + // | j1 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ BX, R11 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ BX, R13 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R14 + ADCQ $0x00, DX + ADDQ BX, R14 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R15 + ADCQ DX, CX + ADDQ BX, R15 + + ADCQ CX, R8 + MOVQ $0x00, CX + ADCQ $0x00, CX + + // | + +/* i4 */ + + MOVQ R11, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + + // | + +/* */ + + // | j0 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, R11 + ADCQ DX, BX + + // | j1 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ BX, R12 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ BX, R13 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R14 + ADCQ $0x00, DX + ADDQ BX, R14 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R15 + ADCQ $0x00, DX + ADDQ BX, R15 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, R8 + ADCQ DX, CX + ADDQ BX, R8 + + MOVQ 80(BP), DI + ADCQ CX, DI + MOVQ $0x00, CX + ADCQ $0x00, CX + +/* i5 */ + + // | | u5 = w5 * inp + MOVQ R12, AX + MULQ ·inp+0(SB) + MOVQ AX, R9 + MOVQ $0x00, BX + +/* */ + + // | j0 + MOVQ ·modulus+0(SB), AX + MULQ R9 + ADDQ AX, R12 + ADCQ DX, BX + + // | j1 + MOVQ ·modulus+8(SB), AX + MULQ R9 + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ BX, R13 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j2 + MOVQ ·modulus+16(SB), AX + MULQ R9 + ADDQ AX, R14 + ADCQ $0x00, DX + ADDQ BX, R14 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j3 + MOVQ ·modulus+24(SB), AX + MULQ R9 + ADDQ AX, R15 + ADCQ $0x00, DX + ADDQ BX, R15 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j4 + MOVQ ·modulus+32(SB), AX + MULQ R9 + ADDQ AX, R8 + ADCQ $0x00, DX + ADDQ BX, R8 + MOVQ $0x00, BX + ADCQ DX, BX + + // | j5 + MOVQ ·modulus+40(SB), AX + MULQ R9 + ADDQ AX, DI + ADCQ DX, CX + ADDQ BX, DI + + ADCQ 88(BP), CX + +/* modular reduction */ + + MOVQ R13, R10 + SUBQ ·modulus+0(SB), R10 + MOVQ R14, R11 + SBBQ ·modulus+8(SB), R11 + MOVQ R15, R12 + SBBQ ·modulus+16(SB), R12 + MOVQ R8, AX + SBBQ ·modulus+24(SB), AX + MOVQ DI, BX + SBBQ ·modulus+32(SB), BX + MOVQ CX, R9 + SBBQ ·modulus+40(SB), R9 + // | + +/* out */ + + MOVQ a+0(FP), SI + CMOVQCC R10, R13 + MOVQ R13, (SI) + CMOVQCC R11, R14 + MOVQ R14, 8(SI) + CMOVQCC R12, R15 + MOVQ R15, 16(SI) + CMOVQCC AX, R8 + MOVQ R8, 24(SI) + CMOVQCC BX, DI + MOVQ DI, 32(SI) + CMOVQCC R9, CX + MOVQ CX, 40(SI) + RET + + // | + +/* end */ \ No newline at end of file diff --git a/crypto/bls12381/fp_fallback.go b/crypto/bls12381/fp_fallback.go new file mode 100644 index 0000000000..3595448615 --- /dev/null +++ b/crypto/bls12381/fp_fallback.go @@ -0,0 +1,1371 @@ +// +build !amd64 generic + +// Copyright 2020 ConsenSys Software Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by goff (v0.3.5) DO NOT EDIT + +// /!\ WARNING /!\ +// this code has not been audited and is provided as-is. In particular, +// there is no security guarantees such as constant time implementation +// or side-channel attack resistance +// /!\ WARNING /!\ + +package bls12381 + +import "math/bits" + +func add(z, x, y *fe) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], y[0], 0) + z[1], carry = bits.Add64(x[1], y[1], carry) + z[2], carry = bits.Add64(x[2], y[2], carry) + z[3], carry = bits.Add64(x[3], y[3], carry) + z[4], carry = bits.Add64(x[4], y[4], carry) + z[5], _ = bits.Add64(x[5], y[5], carry) + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) + z[1], b = bits.Sub64(z[1], 2210141511517208575, b) + z[2], b = bits.Sub64(z[2], 7435674573564081700, b) + z[3], b = bits.Sub64(z[3], 7239337960414712511, b) + z[4], b = bits.Sub64(z[4], 5412103778470702295, b) + z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) + } +} + +func addAssign(z, y *fe) { + var carry uint64 + + z[0], carry = bits.Add64(z[0], y[0], 0) + z[1], carry = bits.Add64(z[1], y[1], carry) + z[2], carry = bits.Add64(z[2], y[2], carry) + z[3], carry = bits.Add64(z[3], y[3], carry) + z[4], carry = bits.Add64(z[4], y[4], carry) + z[5], _ = bits.Add64(z[5], y[5], carry) + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) + z[1], b = bits.Sub64(z[1], 2210141511517208575, b) + z[2], b = bits.Sub64(z[2], 7435674573564081700, b) + z[3], b = bits.Sub64(z[3], 7239337960414712511, b) + z[4], b = bits.Sub64(z[4], 5412103778470702295, b) + z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) + } +} + +func ladd(z, x, y *fe) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], y[0], 0) + z[1], carry = bits.Add64(x[1], y[1], carry) + z[2], carry = bits.Add64(x[2], y[2], carry) + z[3], carry = bits.Add64(x[3], y[3], carry) + z[4], carry = bits.Add64(x[4], y[4], carry) + z[5], _ = bits.Add64(x[5], y[5], carry) +} + +func laddAssign(z, y *fe) { + var carry uint64 + + z[0], carry = bits.Add64(z[0], y[0], 0) + z[1], carry = bits.Add64(z[1], y[1], carry) + z[2], carry = bits.Add64(z[2], y[2], carry) + z[3], carry = bits.Add64(z[3], y[3], carry) + z[4], carry = bits.Add64(z[4], y[4], carry) + z[5], _ = bits.Add64(z[5], y[5], carry) +} + +func double(z, x *fe) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], x[0], 0) + z[1], carry = bits.Add64(x[1], x[1], carry) + z[2], carry = bits.Add64(x[2], x[2], carry) + z[3], carry = bits.Add64(x[3], x[3], carry) + z[4], carry = bits.Add64(x[4], x[4], carry) + z[5], _ = bits.Add64(x[5], x[5], carry) + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) + z[1], b = bits.Sub64(z[1], 2210141511517208575, b) + z[2], b = bits.Sub64(z[2], 7435674573564081700, b) + z[3], b = bits.Sub64(z[3], 7239337960414712511, b) + z[4], b = bits.Sub64(z[4], 5412103778470702295, b) + z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) + } +} + +func doubleAssign(z *fe) { + var carry uint64 + + z[0], carry = bits.Add64(z[0], z[0], 0) + z[1], carry = bits.Add64(z[1], z[1], carry) + z[2], carry = bits.Add64(z[2], z[2], carry) + z[3], carry = bits.Add64(z[3], z[3], carry) + z[4], carry = bits.Add64(z[4], z[4], carry) + z[5], _ = bits.Add64(z[5], z[5], carry) + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) + z[1], b = bits.Sub64(z[1], 2210141511517208575, b) + z[2], b = bits.Sub64(z[2], 7435674573564081700, b) + z[3], b = bits.Sub64(z[3], 7239337960414712511, b) + z[4], b = bits.Sub64(z[4], 5412103778470702295, b) + z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) + } +} + +func ldouble(z, x *fe) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], x[0], 0) + z[1], carry = bits.Add64(x[1], x[1], carry) + z[2], carry = bits.Add64(x[2], x[2], carry) + z[3], carry = bits.Add64(x[3], x[3], carry) + z[4], carry = bits.Add64(x[4], x[4], carry) + z[5], _ = bits.Add64(x[5], x[5], carry) +} + +func sub(z, x, y *fe) { + var b uint64 + z[0], b = bits.Sub64(x[0], y[0], 0) + z[1], b = bits.Sub64(x[1], y[1], b) + z[2], b = bits.Sub64(x[2], y[2], b) + z[3], b = bits.Sub64(x[3], y[3], b) + z[4], b = bits.Sub64(x[4], y[4], b) + z[5], b = bits.Sub64(x[5], y[5], b) + if b != 0 { + var c uint64 + z[0], c = bits.Add64(z[0], 13402431016077863595, 0) + z[1], c = bits.Add64(z[1], 2210141511517208575, c) + z[2], c = bits.Add64(z[2], 7435674573564081700, c) + z[3], c = bits.Add64(z[3], 7239337960414712511, c) + z[4], c = bits.Add64(z[4], 5412103778470702295, c) + z[5], _ = bits.Add64(z[5], 1873798617647539866, c) + } +} + +func subAssign(z, y *fe) { + var b uint64 + z[0], b = bits.Sub64(z[0], y[0], 0) + z[1], b = bits.Sub64(z[1], y[1], b) + z[2], b = bits.Sub64(z[2], y[2], b) + z[3], b = bits.Sub64(z[3], y[3], b) + z[4], b = bits.Sub64(z[4], y[4], b) + z[5], b = bits.Sub64(z[5], y[5], b) + if b != 0 { + var c uint64 + z[0], c = bits.Add64(z[0], 13402431016077863595, 0) + z[1], c = bits.Add64(z[1], 2210141511517208575, c) + z[2], c = bits.Add64(z[2], 7435674573564081700, c) + z[3], c = bits.Add64(z[3], 7239337960414712511, c) + z[4], c = bits.Add64(z[4], 5412103778470702295, c) + z[5], _ = bits.Add64(z[5], 1873798617647539866, c) + } +} + +func lsubAssign(z, y *fe) { + var b uint64 + z[0], b = bits.Sub64(z[0], y[0], 0) + z[1], b = bits.Sub64(z[1], y[1], b) + z[2], b = bits.Sub64(z[2], y[2], b) + z[3], b = bits.Sub64(z[3], y[3], b) + z[4], b = bits.Sub64(z[4], y[4], b) + z[5], b = bits.Sub64(z[5], y[5], b) +} + +func neg(z, x *fe) { + if x.isZero() { + z.zero() + return + } + var borrow uint64 + z[0], borrow = bits.Sub64(13402431016077863595, x[0], 0) + z[1], borrow = bits.Sub64(2210141511517208575, x[1], borrow) + z[2], borrow = bits.Sub64(7435674573564081700, x[2], borrow) + z[3], borrow = bits.Sub64(7239337960414712511, x[3], borrow) + z[4], borrow = bits.Sub64(5412103778470702295, x[4], borrow) + z[5], _ = bits.Sub64(1873798617647539866, x[5], borrow) +} + +func mul(z, x, y *fe) { + + var t [6]uint64 + var c [3]uint64 + { + // round 0 + v := x[0] + c[1], c[0] = bits.Mul64(v, y[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd1(v, y[1], c[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd1(v, y[2], c[1]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd1(v, y[3], c[1]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd1(v, y[4], c[1]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd1(v, y[5], c[1]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 1 + v := x[1] + c[1], c[0] = madd1(v, y[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, y[1], c[1], t[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, y[2], c[1], t[2]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, y[3], c[1], t[3]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, y[4], c[1], t[4]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, y[5], c[1], t[5]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 2 + v := x[2] + c[1], c[0] = madd1(v, y[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, y[1], c[1], t[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, y[2], c[1], t[2]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, y[3], c[1], t[3]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, y[4], c[1], t[4]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, y[5], c[1], t[5]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 3 + v := x[3] + c[1], c[0] = madd1(v, y[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, y[1], c[1], t[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, y[2], c[1], t[2]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, y[3], c[1], t[3]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, y[4], c[1], t[4]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, y[5], c[1], t[5]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 4 + v := x[4] + c[1], c[0] = madd1(v, y[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, y[1], c[1], t[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, y[2], c[1], t[2]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, y[3], c[1], t[3]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, y[4], c[1], t[4]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, y[5], c[1], t[5]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 5 + v := x[5] + c[1], c[0] = madd1(v, y[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, y[1], c[1], t[1]) + c[2], z[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, y[2], c[1], t[2]) + c[2], z[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, y[3], c[1], t[3]) + c[2], z[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, y[4], c[1], t[4]) + c[2], z[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, y[5], c[1], t[5]) + z[5], z[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) + z[1], b = bits.Sub64(z[1], 2210141511517208575, b) + z[2], b = bits.Sub64(z[2], 7435674573564081700, b) + z[3], b = bits.Sub64(z[3], 7239337960414712511, b) + z[4], b = bits.Sub64(z[4], 5412103778470702295, b) + z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) + } +} + +func square(z, x *fe) { + + var t [6]uint64 + var c [3]uint64 + { + // round 0 + v := x[0] + c[1], c[0] = bits.Mul64(v, x[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd1(v, x[1], c[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd1(v, x[2], c[1]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd1(v, x[3], c[1]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd1(v, x[4], c[1]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd1(v, x[5], c[1]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 1 + v := x[1] + c[1], c[0] = madd1(v, x[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, x[1], c[1], t[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, x[2], c[1], t[2]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, x[3], c[1], t[3]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, x[4], c[1], t[4]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, x[5], c[1], t[5]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 2 + v := x[2] + c[1], c[0] = madd1(v, x[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, x[1], c[1], t[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, x[2], c[1], t[2]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, x[3], c[1], t[3]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, x[4], c[1], t[4]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, x[5], c[1], t[5]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 3 + v := x[3] + c[1], c[0] = madd1(v, x[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, x[1], c[1], t[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, x[2], c[1], t[2]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, x[3], c[1], t[3]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, x[4], c[1], t[4]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, x[5], c[1], t[5]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 4 + v := x[4] + c[1], c[0] = madd1(v, x[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, x[1], c[1], t[1]) + c[2], t[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, x[2], c[1], t[2]) + c[2], t[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, x[3], c[1], t[3]) + c[2], t[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, x[4], c[1], t[4]) + c[2], t[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, x[5], c[1], t[5]) + t[5], t[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + { + // round 5 + v := x[5] + c[1], c[0] = madd1(v, x[0], t[0]) + m := c[0] * 9940570264628428797 + c[2] = madd0(m, 13402431016077863595, c[0]) + c[1], c[0] = madd2(v, x[1], c[1], t[1]) + c[2], z[0] = madd2(m, 2210141511517208575, c[2], c[0]) + c[1], c[0] = madd2(v, x[2], c[1], t[2]) + c[2], z[1] = madd2(m, 7435674573564081700, c[2], c[0]) + c[1], c[0] = madd2(v, x[3], c[1], t[3]) + c[2], z[2] = madd2(m, 7239337960414712511, c[2], c[0]) + c[1], c[0] = madd2(v, x[4], c[1], t[4]) + c[2], z[3] = madd2(m, 5412103778470702295, c[2], c[0]) + c[1], c[0] = madd2(v, x[5], c[1], t[5]) + z[5], z[4] = madd3(m, 1873798617647539866, c[0], c[2], c[1]) + } + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[5] < 1873798617647539866 || (z[5] == 1873798617647539866 && (z[4] < 5412103778470702295 || (z[4] == 5412103778470702295 && (z[3] < 7239337960414712511 || (z[3] == 7239337960414712511 && (z[2] < 7435674573564081700 || (z[2] == 7435674573564081700 && (z[1] < 2210141511517208575 || (z[1] == 2210141511517208575 && (z[0] < 13402431016077863595))))))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 13402431016077863595, 0) + z[1], b = bits.Sub64(z[1], 2210141511517208575, b) + z[2], b = bits.Sub64(z[2], 7435674573564081700, b) + z[3], b = bits.Sub64(z[3], 7239337960414712511, b) + z[4], b = bits.Sub64(z[4], 5412103778470702295, b) + z[5], _ = bits.Sub64(z[5], 1873798617647539866, b) + } +} + +func wadd(z, x, y *wfe) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], y[0], 0) + z[1], carry = bits.Add64(x[1], y[1], carry) + z[2], carry = bits.Add64(x[2], y[2], carry) + z[3], carry = bits.Add64(x[3], y[3], carry) + z[4], carry = bits.Add64(x[4], y[4], carry) + z[5], carry = bits.Add64(x[5], y[5], carry) + z[6], carry = bits.Add64(x[6], y[6], carry) + z[7], carry = bits.Add64(x[7], y[7], carry) + z[8], carry = bits.Add64(x[8], y[8], carry) + z[9], carry = bits.Add64(x[9], y[9], carry) + z[10], carry = bits.Add64(x[10], y[10], carry) + z[11], _ = bits.Add64(x[11], y[11], carry) + + if !(z[11] < 1873798617647539866 || (z[11] == 1873798617647539866 && (z[10] < 5412103778470702295 || (z[10] == 5412103778470702295 && (z[9] < 7239337960414712511 || (z[9] == 7239337960414712511 && (z[8] < 7435674573564081700 || (z[8] == 7435674573564081700 && (z[7] < 2210141511517208575 || (z[7] == 2210141511517208575 && (z[6] < 13402431016077863595))))))))))) { + var b uint64 + z[6], b = bits.Sub64(z[6], 13402431016077863595, 0) + z[7], b = bits.Sub64(z[7], 2210141511517208575, b) + z[8], b = bits.Sub64(z[8], 7435674573564081700, b) + z[9], b = bits.Sub64(z[9], 7239337960414712511, b) + z[10], b = bits.Sub64(z[10], 5412103778470702295, b) + z[11], _ = bits.Sub64(z[11], 1873798617647539866, b) + } +} + +func waddAssign(x, y *wfe) { + wadd(x, x, y) +} + +func lwadd(z, x, y *wfe) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], y[0], 0) + z[1], carry = bits.Add64(x[1], y[1], carry) + z[2], carry = bits.Add64(x[2], y[2], carry) + z[3], carry = bits.Add64(x[3], y[3], carry) + z[4], carry = bits.Add64(x[4], y[4], carry) + z[5], carry = bits.Add64(x[5], y[5], carry) + z[6], carry = bits.Add64(x[6], y[6], carry) + z[7], carry = bits.Add64(x[7], y[7], carry) + z[8], carry = bits.Add64(x[8], y[8], carry) + z[9], carry = bits.Add64(x[9], y[9], carry) + z[10], carry = bits.Add64(x[10], y[10], carry) + z[11], _ = bits.Add64(x[11], y[11], carry) +} + +func lwaddAssign(x, y *wfe) { + lwadd(x, x, y) +} + +func wsub(z, x, y *wfe) { + var b uint64 + z[0], b = bits.Sub64(x[0], y[0], 0) + z[1], b = bits.Sub64(x[1], y[1], b) + z[2], b = bits.Sub64(x[2], y[2], b) + z[3], b = bits.Sub64(x[3], y[3], b) + z[4], b = bits.Sub64(x[4], y[4], b) + z[5], b = bits.Sub64(x[5], y[5], b) + z[6], b = bits.Sub64(x[6], y[6], b) + z[7], b = bits.Sub64(x[7], y[7], b) + z[8], b = bits.Sub64(x[8], y[8], b) + z[9], b = bits.Sub64(x[9], y[9], b) + z[10], b = bits.Sub64(x[10], y[10], b) + z[11], b = bits.Sub64(x[11], y[11], b) + if b != 0 { + var c uint64 + z[6], c = bits.Add64(z[6], 13402431016077863595, 0) + z[7], c = bits.Add64(z[7], 2210141511517208575, c) + z[8], c = bits.Add64(z[8], 7435674573564081700, c) + z[9], c = bits.Add64(z[9], 7239337960414712511, c) + z[10], c = bits.Add64(z[10], 5412103778470702295, c) + z[11], _ = bits.Add64(z[11], 1873798617647539866, c) + } +} + +func wsubAssign(x, y *wfe) { + wsub(x, x, y) +} + +func lwsub(z, x, y *wfe) { + var b uint64 + z[0], b = bits.Sub64(x[0], y[0], 0) + z[1], b = bits.Sub64(x[1], y[1], b) + z[2], b = bits.Sub64(x[2], y[2], b) + z[3], b = bits.Sub64(x[3], y[3], b) + z[4], b = bits.Sub64(x[4], y[4], b) + z[5], b = bits.Sub64(x[5], y[5], b) + z[6], b = bits.Sub64(x[6], y[6], b) + z[7], b = bits.Sub64(x[7], y[7], b) + z[8], b = bits.Sub64(x[8], y[8], b) + z[9], b = bits.Sub64(x[9], y[9], b) + z[10], b = bits.Sub64(x[10], y[10], b) + z[11], b = bits.Sub64(x[11], y[11], b) +} + +func lwsubAssign(x, y *wfe) { + lwsub(x, x, y) +} + +func wdouble(z, x *wfe) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], x[0], 0) + z[1], carry = bits.Add64(x[1], x[1], carry) + z[2], carry = bits.Add64(x[2], x[2], carry) + z[3], carry = bits.Add64(x[3], x[3], carry) + z[4], carry = bits.Add64(x[4], x[4], carry) + z[5], carry = bits.Add64(x[5], x[5], carry) + z[6], carry = bits.Add64(x[6], x[6], carry) + z[7], carry = bits.Add64(x[7], x[7], carry) + z[8], carry = bits.Add64(x[8], x[8], carry) + z[9], carry = bits.Add64(x[9], x[9], carry) + z[10], carry = bits.Add64(x[10], x[10], carry) + z[11], _ = bits.Add64(x[11], x[11], carry) + + if !(z[11] < 1873798617647539866 || (z[11] == 1873798617647539866 && (z[10] < 5412103778470702295 || (z[10] == 5412103778470702295 && (z[9] < 7239337960414712511 || (z[9] == 7239337960414712511 && (z[8] < 7435674573564081700 || (z[8] == 7435674573564081700 && (z[7] < 2210141511517208575 || (z[7] == 2210141511517208575 && (z[6] < 13402431016077863595))))))))))) { + var b uint64 + z[6], b = bits.Sub64(z[6], 13402431016077863595, 0) + z[7], b = bits.Sub64(z[7], 2210141511517208575, b) + z[8], b = bits.Sub64(z[8], 7435674573564081700, b) + z[9], b = bits.Sub64(z[9], 7239337960414712511, b) + z[10], b = bits.Sub64(z[10], 5412103778470702295, b) + z[11], _ = bits.Sub64(z[11], 1873798617647539866, b) + } +} + +func wdoubleAssign(x *wfe) { + wdouble(x, x) +} + +func lwdouble(z, x *wfe) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], x[0], 0) + z[1], carry = bits.Add64(x[1], x[1], carry) + z[2], carry = bits.Add64(x[2], x[2], carry) + z[3], carry = bits.Add64(x[3], x[3], carry) + z[4], carry = bits.Add64(x[4], x[4], carry) + z[5], carry = bits.Add64(x[5], x[5], carry) + z[6], carry = bits.Add64(x[6], x[6], carry) + z[7], carry = bits.Add64(x[7], x[7], carry) + z[8], carry = bits.Add64(x[8], x[8], carry) + z[9], carry = bits.Add64(x[9], x[9], carry) + z[10], carry = bits.Add64(x[10], x[10], carry) + z[11], _ = bits.Add64(x[11], x[11], carry) +} + +func fromWide(c *fe, w *wfe) { + montRed(c, w) +} + +func wmul(w *wfe, a, b *fe) { + + var w0, w1, w2, w3, w4, w5, w6, w7, w8, w9, w10, w11 uint64 + var a0 = a[0] + var a1 = a[1] + var a2 = a[2] + var a3 = a[3] + var a4 = a[4] + var a5 = a[5] + var b0 = b[0] + var b1 = b[1] + var b2 = b[2] + var b3 = b[3] + var b4 = b[4] + var b5 = b[5] + var u, v, c, t uint64 + + { + // i = 0, j = 0 + c, w0 = bits.Mul64(a0, b0) + + // i = 0, j = 1 + u, v = bits.Mul64(a1, b0) + w1 = v + c + c = u + (v&c|(v|c)&^w1)>>63 + + // i = 0, j = 2 + u, v = bits.Mul64(a2, b0) + w2 = v + c + c = u + (v&c|(v|c)&^w2)>>63 + + // i = 0, j = 3 + u, v = bits.Mul64(a3, b0) + w3 = v + c + c = u + (v&c|(v|c)&^w3)>>63 + + // i = 0, j = 4 + u, v = bits.Mul64(a4, b0) + w4 = v + c + c = u + (v&c|(v|c)&^w4)>>63 + + // i = 0, j = 5 + u, v = bits.Mul64(a5, b0) + w5 = v + c + w6 = u + (v&c|(v|c)&^w5)>>63 + } + + { + + // i = 1, j = 0 + c, v = bits.Mul64(a0, b1) + t = v + w1 + c += (v&w1 | (v|w1)&^t) >> 63 + w1 = t + + // i = 1, j = 1 + u, v = bits.Mul64(a1, b1) + t = v + w2 + u += (v&w2 | (v|w2)&^t) >> 63 + w2 = t + c + c = u + (t&c|(t|c)&^w2)>>63 + + // i = 1, j = 2 + u, v = bits.Mul64(a2, b1) + t = v + w3 + u += (v&w3 | (v|w3)&^t) >> 63 + w3 = t + c + c = u + (t&c|(t|c)&^w3)>>63 + + // i = 1, j = 3 + u, v = bits.Mul64(a3, b1) + t = v + w4 + u += (v&w4 | (v|w4)&^t) >> 63 + w4 = t + c + c = u + (t&c|(t|c)&^w4)>>63 + + // i = 1, j = 4 + u, v = bits.Mul64(a4, b1) + t = v + w5 + u += (v&w5 | (v|w5)&^t) >> 63 + w5 = t + c + c = u + (t&c|(t|c)&^w5)>>63 + + // i = 1, j = 5 + u, v = bits.Mul64(a5, b1) + t = v + w6 + u += (v&w6 | (v|w6)&^t) >> 63 + w6 = t + c + w7 = u + (t&c|(t|c)&^w6)>>63 + } + + { + // i = 2, j = 0 + c, v = bits.Mul64(a0, b2) + t = v + w2 + c += (v&w2 | (v|w2)&^t) >> 63 + w2 = t + + // i = 2, j = 1 + u, v = bits.Mul64(a1, b2) + t = v + w3 + u += (v&w3 | (v|w3)&^t) >> 63 + w3 = t + c + c = u + (t&c|(t|c)&^w3)>>63 + + // i = 2, j = 2 + u, v = bits.Mul64(a2, b2) + t = v + w4 + u += (v&w4 | (v|w4)&^t) >> 63 + w4 = t + c + c = u + (t&c|(t|c)&^w4)>>63 + + // i = 2, j = 3 + u, v = bits.Mul64(a3, b2) + t = v + w5 + u += (v&w5 | (v|w5)&^t) >> 63 + w5 = t + c + c = u + (t&c|(t|c)&^w5)>>63 + + // i = 2, j = 4 + u, v = bits.Mul64(a4, b2) + t = v + w6 + u += (v&w6 | (v|w6)&^t) >> 63 + w6 = t + c + c = u + (t&c|(t|c)&^w6)>>63 + + // i = 2, j = 5 + u, v = bits.Mul64(a5, b2) + t = v + w7 + u += (v&w7 | (v|w7)&^t) >> 63 + w7 = t + c + w8 = u + (t&c|(t|c)&^w7)>>63 + } + + { + // i = 3, j = 0 + c, v = bits.Mul64(a0, b3) + t = v + w3 + c += (v&w3 | (v|w3)&^t) >> 63 + w3 = t + + // i = 3, j = 1 + u, v = bits.Mul64(a1, b3) + t = v + w4 + u += (v&w4 | (v|w4)&^t) >> 63 + w4 = t + c + c = u + (t&c|(t|c)&^w4)>>63 + + // i = 3, j = 2 + u, v = bits.Mul64(a2, b3) + t = v + w5 + u += (v&w5 | (v|w5)&^t) >> 63 + w5 = t + c + c = u + (t&c|(t|c)&^w5)>>63 + + // i = 3, j = 3 + u, v = bits.Mul64(a3, b3) + t = v + w6 + u += (v&w6 | (v|w6)&^t) >> 63 + w6 = t + c + c = u + (t&c|(t|c)&^w6)>>63 + + // i = 3, j = 4 + u, v = bits.Mul64(a4, b3) + t = v + w7 + u += (v&w7 | (v|w7)&^t) >> 63 + w7 = t + c + c = u + (t&c|(t|c)&^w7)>>63 + + // i = 3, j = 5 + u, v = bits.Mul64(a5, b3) + t = v + w8 + u += (v&w8 | (v|w8)&^t) >> 63 + w8 = t + c + w9 = u + (t&c|(t|c)&^w8)>>63 + } + + { + // i = 4, j = 0 + c, v = bits.Mul64(a0, b4) + t = v + w4 + c += (v&w4 | (v|w4)&^t) >> 63 + w4 = t + + // i = 4, j = 1 + u, v = bits.Mul64(a1, b4) + t = v + w5 + u += (v&w5 | (v|w5)&^t) >> 63 + w5 = t + c + c = u + (t&c|(t|c)&^w5)>>63 + + // i = 4, j = 2 + u, v = bits.Mul64(a2, b4) + t = v + w6 + u += (v&w6 | (v|w6)&^t) >> 63 + w6 = t + c + c = u + (t&c|(t|c)&^w6)>>63 + + // i = 4, j = 3 + u, v = bits.Mul64(a3, b4) + t = v + w7 + u += (v&w7 | (v|w7)&^t) >> 63 + w7 = t + c + c = u + (t&c|(t|c)&^w7)>>63 + + // i = 4, j = 4 + u, v = bits.Mul64(a4, b4) + t = v + w8 + u += (v&w8 | (v|w8)&^t) >> 63 + w8 = t + c + c = u + (t&c|(t|c)&^w8)>>63 + + // i = 4, j = 5 + u, v = bits.Mul64(a5, b4) + t = v + w9 + u += (v&w9 | (v|w9)&^t) >> 63 + w9 = t + c + w10 = u + (t&c|(t|c)&^w9)>>63 + } + + { + // i = 5, j = 0 + c, v = bits.Mul64(a0, b5) + t = v + w5 + c += (v&w5 | (v|w5)&^t) >> 63 + w5 = t + + // i = 5, j = 1 + u, v = bits.Mul64(a1, b5) + t = v + w6 + u += (v&w6 | (v|w6)&^t) >> 63 + w6 = t + c + c = u + (t&c|(t|c)&^w6)>>63 + + // i = 5, j = 2 + u, v = bits.Mul64(a2, b5) + t = v + w7 + u += (v&w7 | (v|w7)&^t) >> 63 + w7 = t + c + c = u + (t&c|(t|c)&^w7)>>63 + + // i = 5, j = 3 + u, v = bits.Mul64(a3, b5) + t = v + w8 + u += (v&w8 | (v|w8)&^t) >> 63 + w8 = t + c + c = u + (t&c|(t|c)&^w8)>>63 + + // i = 5, j = 4 + u, v = bits.Mul64(a4, b5) + t = v + w9 + u += (v&w9 | (v|w9)&^t) >> 63 + w9 = t + c + c = u + (t&c|(t|c)&^w9)>>63 + + // i = 5, j = 5 + u, v = bits.Mul64(a5, b5) + t = v + w10 + u += (v&w10 | (v|w10)&^t) >> 63 + w10 = t + c + w11 = u + (t&c|(t|c)&^w10)>>63 + } + + w[0] = w0 + w[1] = w1 + w[2] = w2 + w[3] = w3 + w[4] = w4 + w[5] = w5 + w[6] = w6 + w[7] = w7 + w[8] = w8 + w[9] = w9 + w[10] = w10 + w[11] = w11 +} + +func montRed(c *fe, w *wfe) { + + // Reduces T as T (R^-1) modp + // Handbook of Applied Cryptography + // Hankerson, Menezes, Vanstone + // Algorithm 14.32 Montgomery reduction + + w0 := w[0] + w1 := w[1] + w2 := w[2] + w3 := w[3] + w4 := w[4] + w5 := w[5] + w6 := w[6] + w7 := w[7] + w8 := w[8] + w9 := w[9] + w10 := w[10] + w11 := w[11] + p0 := modulus[0] + p1 := modulus[1] + p2 := modulus[2] + p3 := modulus[3] + p4 := modulus[4] + p5 := modulus[5] + + var e1, e2, el, res uint64 + var t1, t2, u uint64 + + { + + // i = 0 + u = w0 * inp + // + e1, res = bits.Mul64(u, p0) + t1 = res + w0 + e1 += (res&w0 | (res|w0)&^t1) >> 63 + // + e2, res = bits.Mul64(u, p1) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w1 + e2 += (t1&w1 | (t1|w1)&^t2) >> 63 + w1 = t2 + // + e1, res = bits.Mul64(u, p2) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w2 + e1 += (t1&w2 | (t1|w2)&^t2) >> 63 + w2 = t2 + // + e2, res = bits.Mul64(u, p3) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w3 + e2 += (t1&w3 | (t1|w3)&^t2) >> 63 + w3 = t2 + // + e1, res = bits.Mul64(u, p4) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w4 + e1 += (t1&w4 | (t1|w4)&^t2) >> 63 + w4 = t2 + // + e2, res = bits.Mul64(u, p5) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w5 + e2 += (t1&w5 | (t1|w5)&^t2) >> 63 + w5 = t2 + // + t1 = w6 + el + e1 = (w6&el | (w6|el)&^t1) >> 63 + t2 = t1 + e2 + e1 += (t1&e2 | (t1|e2)&^t2) >> 63 + w6 = t2 + el = e1 + } + + { + // i = 1 + u = w1 * inp + // + e1, res = bits.Mul64(u, p0) + t1 = res + w1 + e1 += (res&w1 | (res|w1)&^t1) >> 63 + // + e2, res = bits.Mul64(u, p1) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w2 + e2 += (t1&w2 | (t1|w2)&^t2) >> 63 + w2 = t2 + // + e1, res = bits.Mul64(u, p2) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w3 + e1 += (t1&w3 | (t1|w3)&^t2) >> 63 + w3 = t2 + // + e2, res = bits.Mul64(u, p3) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w4 + e2 += (t1&w4 | (t1|w4)&^t2) >> 63 + w4 = t2 + // + e1, res = bits.Mul64(u, p4) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w5 + e1 += (t1&w5 | (t1|w5)&^t2) >> 63 + w5 = t2 + // + e2, res = bits.Mul64(u, p5) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w6 + e2 += (t1&w6 | (t1|w6)&^t2) >> 63 + w6 = t2 + // + t1 = w7 + el + e1 = (w7&el | (w7|el)&^t1) >> 63 + t2 = t1 + e2 + e1 += (t1&e2 | (t1|e2)&^t2) >> 63 + w7 = t2 + el = e1 + } + + { + // i = 2 + u = w2 * inp + // + e1, res = bits.Mul64(u, p0) + t1 = res + w2 + e1 += (res&w2 | (res|w2)&^t1) >> 63 + // + e2, res = bits.Mul64(u, p1) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w3 + e2 += (t1&w3 | (t1|w3)&^t2) >> 63 + w3 = t2 + // + e1, res = bits.Mul64(u, p2) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w4 + e1 += (t1&w4 | (t1|w4)&^t2) >> 63 + w4 = t2 + // + e2, res = bits.Mul64(u, p3) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w5 + e2 += (t1&w5 | (t1|w5)&^t2) >> 63 + w5 = t2 + // + e1, res = bits.Mul64(u, p4) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w6 + e1 += (t1&w6 | (t1|w6)&^t2) >> 63 + w6 = t2 + // + e2, res = bits.Mul64(u, p5) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w7 + e2 += (t1&w7 | (t1|w7)&^t2) >> 63 + w7 = t2 + // + t1 = w8 + el + e1 = (w8&el | (w8|el)&^t1) >> 63 + t2 = t1 + e2 + e1 += (t1&e2 | (t1|e2)&^t2) >> 63 + w8 = t2 + el = e1 + } + + { + // i = 3 + u = w3 * inp + // + e1, res = bits.Mul64(u, p0) + t1 = res + w3 + e1 += (res&w3 | (res|w3)&^t1) >> 63 + // + e2, res = bits.Mul64(u, p1) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w4 + e2 += (t1&w4 | (t1|w4)&^t2) >> 63 + w4 = t2 + // + e1, res = bits.Mul64(u, p2) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w5 + e1 += (t1&w5 | (t1|w5)&^t2) >> 63 + w5 = t2 + // + e2, res = bits.Mul64(u, p3) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w6 + e2 += (t1&w6 | (t1|w6)&^t2) >> 63 + w6 = t2 + // + e1, res = bits.Mul64(u, p4) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w7 + e1 += (t1&w7 | (t1|w7)&^t2) >> 63 + w7 = t2 + // + e2, res = bits.Mul64(u, p5) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w8 + e2 += (t1&w8 | (t1|w8)&^t2) >> 63 + w8 = t2 + // + t1 = w9 + el + e1 = (w9&el | (w9|el)&^t1) >> 63 + t2 = t1 + e2 + e1 += (t1&e2 | (t1|e2)&^t2) >> 63 + w9 = t2 + el = e1 + } + + { + // i = 4 + u = w4 * inp + // + e1, res = bits.Mul64(u, p0) + t1 = res + w4 + e1 += (res&w4 | (res|w4)&^t1) >> 63 + // + e2, res = bits.Mul64(u, p1) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w5 + e2 += (t1&w5 | (t1|w5)&^t2) >> 63 + w5 = t2 + // + e1, res = bits.Mul64(u, p2) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w6 + e1 += (t1&w6 | (t1|w6)&^t2) >> 63 + w6 = t2 + // + e2, res = bits.Mul64(u, p3) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w7 + e2 += (t1&w7 | (t1|w7)&^t2) >> 63 + w7 = t2 + // + e1, res = bits.Mul64(u, p4) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w8 + e1 += (t1&w8 | (t1|w8)&^t2) >> 63 + w8 = t2 + // + e2, res = bits.Mul64(u, p5) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w9 + e2 += (t1&w9 | (t1|w9)&^t2) >> 63 + w9 = t2 + // + t1 = w10 + el + e1 = (w10&el | (w10|el)&^t1) >> 63 + t2 = t1 + e2 + e1 += (t1&e2 | (t1|e2)&^t2) >> 63 + w10 = t2 + el = e1 + } + + { + // i = 5 + u = w5 * inp + // + e1, res = bits.Mul64(u, p0) + t1 = res + w5 + e1 += (res&w5 | (res|w5)&^t1) >> 63 + // + e2, res = bits.Mul64(u, p1) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w6 + e2 += (t1&w6 | (t1|w6)&^t2) >> 63 + w6 = t2 + // + e1, res = bits.Mul64(u, p2) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w7 + e1 += (t1&w7 | (t1|w7)&^t2) >> 63 + w7 = t2 + // + e2, res = bits.Mul64(u, p3) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w8 + e2 += (t1&w8 | (t1|w8)&^t2) >> 63 + w8 = t2 + // + e1, res = bits.Mul64(u, p4) + t1 = res + e2 + e1 += (res&e2 | (res|e2)&^t1) >> 63 + t2 = t1 + w9 + e1 += (t1&w9 | (t1|w9)&^t2) >> 63 + w9 = t2 + // + e2, res = bits.Mul64(u, p5) + t1 = res + e1 + e2 += (res&e1 | (res|e1)&^t1) >> 63 + t2 = t1 + w10 + e2 += (t1&w10 | (t1|w10)&^t2) >> 63 + w10 = t2 + // + t1 = w11 + el + e1 = (w11&el | (w11|el)&^t1) >> 63 + t2 = t1 + e2 + e1 += (t1&e2 | (t1|e2)&^t2) >> 63 + w11 = t2 + } + + e1-- + c[0] = w6 - ((p0) & ^e1) + e2 = (^w6&p0 | (^w6|p0)&c[0]) >> 63 + c[1] = w7 - ((p1 + e2) & ^e1) + e2 = (^w7&p1 | (^w7|p1)&c[1]) >> 63 + c[2] = w8 - ((p2 + e2) & ^e1) + e2 = (^w8&p2 | (^w8|p2)&c[2]) >> 63 + c[3] = w9 - ((p3 + e2) & ^e1) + e2 = (^w9&p3 | (^w9|p3)&c[3]) >> 63 + c[4] = w10 - ((p4 + e2) & ^e1) + e2 = (^w10&p4 | (^w10|p4)&c[4]) >> 63 + c[5] = w11 - ((p5 + e2) & ^e1) + + sub(c, c, &modulus) +} + +func fp2Add(c, a, b *fe2) { + add(&c[0], &a[0], &b[0]) + add(&c[1], &a[1], &b[1]) +} + +func fp2AddAssign(a, b *fe2) { + addAssign(&a[0], &b[0]) + addAssign(&a[1], &b[1]) +} + +func fp2Ladd(c, a, b *fe2) { + ladd(&c[0], &a[0], &b[0]) + ladd(&c[1], &a[1], &b[1]) +} + +func fp2LaddAssign(a, b *fe2) { + laddAssign(&a[0], &b[0]) + laddAssign(&a[1], &b[1]) +} + +func fp2Double(c, a *fe2) { + double(&c[0], &a[0]) + double(&c[1], &a[1]) +} + +func fp2DoubleAssign(a *fe2) { + doubleAssign(&a[0]) + doubleAssign(&a[1]) +} + +func fp2Ldouble(c, a *fe2) { + ldouble(&c[0], &a[0]) + ldouble(&c[1], &a[1]) +} + +func fp2Sub(c, a, b *fe2) { + sub(&c[0], &a[0], &b[0]) + sub(&c[1], &a[1], &b[1]) +} + +func fp2SubAssign(c, a *fe2) { + subAssign(&c[0], &a[0]) + subAssign(&c[1], &a[1]) +} + +func mulByNonResidue(c, a *fe2) { + t := new(fe) + sub(t, &a[0], &a[1]) + add(&c[1], &a[0], &a[1]) + c[0].set(t) +} + +func mulByNonResidueAssign(a *fe2) { + t := new(fe) + sub(t, &a[0], &a[1]) + add(&a[1], &a[0], &a[1]) + a[0].set(t) +} + +func wfp2Add(c, a, b *wfe2) { + wadd(&c[0], &a[0], &b[0]) + wadd(&c[1], &a[1], &b[1]) +} + +func wfp2AddAssign(c, a *wfe2) { + waddAssign(&c[0], &a[0]) + waddAssign(&c[1], &a[1]) +} + +func wfp2Ladd(c, a, b *wfe2) { + lwadd(&c[0], &a[0], &b[0]) + lwadd(&c[1], &a[1], &b[1]) +} + +func wfp2LaddAssign(a, b *wfe2) { + lwaddAssign(&a[0], &b[0]) + lwaddAssign(&a[1], &b[1]) +} + +func wfp2AddMixed(c, a, b *wfe2) { + wadd(&c[0], &a[0], &b[0]) + lwadd(&c[1], &a[1], &b[1]) +} + +func wfp2AddMixedAssign(a, b *wfe2) { + waddAssign(&a[0], &b[0]) + lwaddAssign(&a[1], &b[1]) +} + +func wfp2Sub(c, a, b *wfe2) { + wsub(&c[0], &a[0], &b[0]) + wsub(&c[1], &a[1], &b[1]) +} + +func wfp2SubAssign(a, b *wfe2) { + wsub(&a[0], &a[0], &b[0]) + wsub(&a[1], &a[1], &b[1]) +} + +func wfp2SubMixed(c, a, b *wfe2) { + wsub(&c[0], &a[0], &b[0]) + lwsub(&c[1], &a[1], &b[1]) +} + +func wfp2SubMixedAssign(a, b *wfe2) { + wsubAssign(&a[0], &b[0]) + lwsubAssign(&a[1], &b[1]) +} + +func wfp2Double(c, a *wfe2) { + wdouble(&c[0], &a[0]) + wdouble(&c[1], &a[1]) +} + +func wfp2DoubleAssign(a *wfe2) { + wdoubleAssign(&a[0]) + wdoubleAssign(&a[1]) +} + +func wfp2MulByNonResidue(c, a *wfe2) { + wt0 := &wfe{} + wadd(wt0, &a[0], &a[1]) + wsub(&c[0], &a[0], &a[1]) + c[1].set(wt0) +} + +func wfp2MulByNonResidueAssign(a *wfe2) { + wt0 := &wfe{} + wadd(wt0, &a[0], &a[1]) + wsub(&a[0], &a[0], &a[1]) + a[1].set(wt0) +} + +var wfp2Mul func(c *wfe2, a, b *fe2) = wfp2MulGeneric +var wfp2Square func(c *wfe2, a *fe2) = wfp2SquareGeneric diff --git a/crypto/bls12381/fp_test.go b/crypto/bls12381/fp_test.go index 0bad35de16..1286a789d4 100644 --- a/crypto/bls12381/fp_test.go +++ b/crypto/bls12381/fp_test.go @@ -9,16 +9,16 @@ import ( func TestFpSerialization(t *testing.T) { t.Run("zero", func(t *testing.T) { - in := make([]byte, 48) + in := make([]byte, fpByteSize) fe, err := fromBytes(in) if err != nil { t.Fatal(err) } if !fe.isZero() { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } if !bytes.Equal(in, toBytes(fe)) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } }) t.Run("bytes", func(t *testing.T) { @@ -29,7 +29,7 @@ func TestFpSerialization(t *testing.T) { t.Fatal(err) } if !a.equal(b) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } } }) @@ -41,7 +41,7 @@ func TestFpSerialization(t *testing.T) { t.Fatal(err) } if !a.equal(b) { - t.Fatal("bad encoding or decoding") + t.Fatal("encoding or decoding failed") } } }) @@ -53,7 +53,7 @@ func TestFpSerialization(t *testing.T) { t.Fatal(err) } if !a.equal(b) { - t.Fatal("bad encoding or decoding") + t.Fatal("encoding or decoding failed") } } }) @@ -64,32 +64,32 @@ func TestFpAdditionCrossAgainstBigInt(t *testing.T) { a, _ := new(fe).rand(rand.Reader) b, _ := new(fe).rand(rand.Reader) c := new(fe) - big_a := toBig(a) - big_b := toBig(b) + big_a := a.big() + big_b := b.big() big_c := new(big.Int) add(c, a, b) - out_1 := toBytes(c) - out_2 := padBytes(big_c.Add(big_a, big_b).Mod(big_c, modulus.big()).Bytes(), 48) + out_1 := c.bytes() + out_2 := padBytes(big_c.Add(big_a, big_b).Mod(big_c, modulus.big()).Bytes(), fpByteSize) if !bytes.Equal(out_1, out_2) { - t.Fatal("cross test against big.Int is not satisfied A") + t.Fatal("cross test against big.Int is failed A") } double(c, a) - out_1 = toBytes(c) - out_2 = padBytes(big_c.Add(big_a, big_a).Mod(big_c, modulus.big()).Bytes(), 48) + out_1 = c.bytes() + out_2 = padBytes(big_c.Add(big_a, big_a).Mod(big_c, modulus.big()).Bytes(), fpByteSize) if !bytes.Equal(out_1, out_2) { - t.Fatal("cross test against big.Int is not satisfied B") + t.Fatal("cross test against big.Int is failed B") } sub(c, a, b) - out_1 = toBytes(c) - out_2 = padBytes(big_c.Sub(big_a, big_b).Mod(big_c, modulus.big()).Bytes(), 48) + out_1 = c.bytes() + out_2 = padBytes(big_c.Sub(big_a, big_b).Mod(big_c, modulus.big()).Bytes(), fpByteSize) if !bytes.Equal(out_1, out_2) { - t.Fatal("cross test against big.Int is not satisfied C") + t.Fatal("cross test against big.Int is failed C") } neg(c, a) - out_1 = toBytes(c) - out_2 = padBytes(big_c.Neg(big_a).Mod(big_c, modulus.big()).Bytes(), 48) + out_1 = c.bytes() + out_2 = padBytes(big_c.Neg(big_a).Mod(big_c, modulus.big()).Bytes(), fpByteSize) if !bytes.Equal(out_1, out_2) { - t.Fatal("cross test against big.Int is not satisfied D") + t.Fatal("cross test against big.Int is failed D") } } } @@ -98,89 +98,90 @@ func TestFpAdditionCrossAgainstBigIntAssigned(t *testing.T) { for i := 0; i < fuz; i++ { a, _ := new(fe).rand(rand.Reader) b, _ := new(fe).rand(rand.Reader) - big_a, big_b := toBig(a), toBig(b) + big_a, big_b := a.big(), b.big() addAssign(a, b) - out_1 := toBytes(a) - out_2 := padBytes(big_a.Add(big_a, big_b).Mod(big_a, modulus.big()).Bytes(), 48) + out_1 := a.bytes() + out_2 := padBytes(big_a.Add(big_a, big_b).Mod(big_a, modulus.big()).Bytes(), fpByteSize) if !bytes.Equal(out_1, out_2) { - t.Fatal("cross test against big.Int is not satisfied A") + t.Fatal("cross test against big.Int is failed A") } a, _ = new(fe).rand(rand.Reader) - big_a = toBig(a) + big_a = a.big() doubleAssign(a) - out_1 = toBytes(a) - out_2 = padBytes(big_a.Add(big_a, big_a).Mod(big_a, modulus.big()).Bytes(), 48) + out_1 = a.bytes() + out_2 = padBytes(big_a.Add(big_a, big_a).Mod(big_a, modulus.big()).Bytes(), fpByteSize) if !bytes.Equal(out_1, out_2) { - t.Fatal("cross test against big.Int is not satisfied B") + t.Fatal("cross test against big.Int is failed B") } a, _ = new(fe).rand(rand.Reader) b, _ = new(fe).rand(rand.Reader) - big_a, big_b = toBig(a), toBig(b) + big_a, big_b = a.big(), b.big() subAssign(a, b) - out_1 = toBytes(a) - out_2 = padBytes(big_a.Sub(big_a, big_b).Mod(big_a, modulus.big()).Bytes(), 48) + out_1 = a.bytes() + out_2 = padBytes(big_a.Sub(big_a, big_b).Mod(big_a, modulus.big()).Bytes(), fpByteSize) if !bytes.Equal(out_1, out_2) { - t.Fatal("cross test against big.Int is not satisfied A") + t.Fatal("cross test against big.Int is failed A") } } } func TestFpAdditionProperties(t *testing.T) { for i := 0; i < fuz; i++ { + zero := new(fe).zero() a, _ := new(fe).rand(rand.Reader) b, _ := new(fe).rand(rand.Reader) - c_1, c_2 := new(fe), new(fe) - add(c_1, a, zero) - if !c_1.equal(a) { + c1, c2 := new(fe), new(fe) + add(c1, a, zero) + if !c1.equal(a) { t.Fatal("a + 0 == a") } - sub(c_1, a, zero) - if !c_1.equal(a) { + sub(c1, a, zero) + if !c1.equal(a) { t.Fatal("a - 0 == a") } - double(c_1, zero) - if !c_1.equal(zero) { + double(c1, zero) + if !c1.equal(zero) { t.Fatal("2 * 0 == 0") } - neg(c_1, zero) - if !c_1.equal(zero) { + neg(c1, zero) + if !c1.equal(zero) { t.Fatal("-0 == 0") } - sub(c_1, zero, a) - neg(c_2, a) - if !c_1.equal(c_2) { + sub(c1, zero, a) + neg(c2, a) + if !c1.equal(c2) { t.Fatal("0-a == -a") } - double(c_1, a) - add(c_2, a, a) - if !c_1.equal(c_2) { + double(c1, a) + add(c2, a, a) + if !c1.equal(c2) { t.Fatal("2 * a == a + a") } - add(c_1, a, b) - add(c_2, b, a) - if !c_1.equal(c_2) { + add(c1, a, b) + add(c2, b, a) + if !c1.equal(c2) { t.Fatal("a + b = b + a") } - sub(c_1, a, b) - sub(c_2, b, a) - neg(c_2, c_2) - if !c_1.equal(c_2) { + sub(c1, a, b) + sub(c2, b, a) + neg(c2, c2) + if !c1.equal(c2) { t.Fatal("a - b = - ( b - a )") } - c_x, _ := new(fe).rand(rand.Reader) - add(c_1, a, b) - add(c_1, c_1, c_x) - add(c_2, a, c_x) - add(c_2, c_2, b) - if !c_1.equal(c_2) { + cx, _ := new(fe).rand(rand.Reader) + add(c1, a, b) + add(c1, c1, cx) + add(c2, a, cx) + add(c2, c2, b) + if !c1.equal(c2) { t.Fatal("(a + b) + c == (a + c ) + b") } - sub(c_1, a, b) - sub(c_1, c_1, c_x) - sub(c_2, a, c_x) - sub(c_2, c_2, b) - if !c_1.equal(c_2) { + sub(c1, a, b) + sub(c1, c1, cx) + sub(c2, a, cx) + sub(c2, c2, b) + if !c1.equal(c2) { t.Fatal("(a - b) - c == (a - c ) -b") } } @@ -220,20 +221,20 @@ func TestFpAdditionPropertiesAssigned(t *testing.T) { } _, _ = a.rand(rand.Reader) _, _ = b.rand(rand.Reader) - c_1, c_2 := new(fe).set(a), new(fe).set(b) - addAssign(c_1, b) - addAssign(c_2, a) - if !c_1.equal(c_2) { + c1, c2 := new(fe).set(a), new(fe).set(b) + addAssign(c1, b) + addAssign(c2, a) + if !c1.equal(c2) { t.Fatal("a + b = b + a") } _, _ = a.rand(rand.Reader) _, _ = b.rand(rand.Reader) - c_1.set(a) - c_2.set(b) - subAssign(c_1, b) - subAssign(c_2, a) - neg(c_2, c_2) - if !c_1.equal(c_2) { + c1.set(a) + c2.set(b) + subAssign(c1, b) + subAssign(c2, a) + neg(c2, c2) + if !c1.equal(c2) { t.Fatal("a - b = - ( b - a )") } _, _ = a.rand(rand.Reader) @@ -273,7 +274,6 @@ func TestFpLazyOperations(t *testing.T) { mul(c0, c0, c) mul(c1, c1, c) if !c0.equal(c1) { - // l+ operator stands for lazy addition t.Fatal("(a + b) * c == (a l+ b) * c") } _, _ = a.rand(rand.Reader) @@ -308,9 +308,9 @@ func TestFpMultiplicationCrossAgainstBigInt(t *testing.T) { big_c := new(big.Int) mul(c, a, b) out_1 := toBytes(c) - out_2 := padBytes(big_c.Mul(big_a, big_b).Mod(big_c, modulus.big()).Bytes(), 48) + out_2 := padBytes(big_c.Mul(big_a, big_b).Mod(big_c, modulus.big()).Bytes(), fpByteSize) if !bytes.Equal(out_1, out_2) { - t.Fatal("cross test against big.Int is not satisfied") + t.Fatal("cross test against big.Int is failed") } } } @@ -320,26 +320,26 @@ func TestFpMultiplicationProperties(t *testing.T) { a, _ := new(fe).rand(rand.Reader) b, _ := new(fe).rand(rand.Reader) zero, one := new(fe).zero(), new(fe).one() - c_1, c_2 := new(fe), new(fe) - mul(c_1, a, zero) - if !c_1.equal(zero) { + c1, c2 := new(fe), new(fe) + mul(c1, a, zero) + if !c1.equal(zero) { t.Fatal("a * 0 == 0") } - mul(c_1, a, one) - if !c_1.equal(a) { + mul(c1, a, one) + if !c1.equal(a) { t.Fatal("a * 1 == a") } - mul(c_1, a, b) - mul(c_2, b, a) - if !c_1.equal(c_2) { + mul(c1, a, b) + mul(c2, b, a) + if !c1.equal(c2) { t.Fatal("a * b == b * a") } - c_x, _ := new(fe).rand(rand.Reader) - mul(c_1, a, b) - mul(c_1, c_1, c_x) - mul(c_2, c_x, b) - mul(c_2, c_2, a) - if !c_1.equal(c_2) { + cx, _ := new(fe).rand(rand.Reader) + mul(c1, a, b) + mul(c1, c1, cx) + mul(c2, cx, b) + mul(c2, c2, a) + if !c1.equal(c2) { t.Fatal("(a * b) * c == (a * c) * b") } square(a, zero) @@ -351,9 +351,9 @@ func TestFpMultiplicationProperties(t *testing.T) { t.Fatal("1^2 == 1") } _, _ = a.rand(rand.Reader) - square(c_1, a) - mul(c_2, a, a) - if !c_1.equal(c_1) { + square(c1, a) + mul(c2, a, a) + if !c1.equal(c1) { t.Fatal("a^2 == a*a") } } @@ -419,21 +419,52 @@ func TestFpInversion(t *testing.T) { } } +func TestFpBatchInversion(t *testing.T) { + n := 20 + for i := 0; i < n; i++ { + e0 := make([]fe, n) + e1 := make([]fe, n) + for j := 0; j < n; j++ { + if j != i { + e, err := new(fe).rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + e0[j].set(e) + } + inverse(&e1[j], &e0[j]) + } + + inverseBatch(e0) + for j := 0; j < n; j++ { + if !e0[j].equal(&e1[j]) { + t.Fatal("batch inversion failed") + } + } + } +} + func TestFpSquareRoot(t *testing.T) { - r := new(fe) - if sqrt(r, nonResidue1) { + if sqrt(new(fe), nonResidue1) { t.Fatal("non residue cannot have a sqrt") } for i := 0; i < fuz; i++ { a, _ := new(fe).rand(rand.Reader) - aa, rr, r := &fe{}, &fe{}, &fe{} - square(aa, a) - if !sqrt(r, aa) { - t.Fatal("bad sqrt 1") + r0, r1 := new(fe), new(fe) + d0 := sqrt(r0, a) + d1 := _sqrt(r1, a) + if d0 != d1 { + t.Fatal("sqrt decision failed") } - square(rr, r) - if !rr.equal(aa) { - t.Fatal("bad sqrt 2") + if d0 { + square(r0, r0) + square(r1, r1) + if !r0.equal(r1) { + t.Fatal("sqrt failed") + } + if !r0.equal(a) { + t.Fatal("sqrt failed") + } } } } @@ -451,7 +482,7 @@ func TestFpNonResidue(t *testing.T) { for i := 0; i < fuz; i++ { a, _ := new(fe).rand(rand.Reader) square(a, a) - if isQuadraticNonResidue(new(fe).one()) { + if isQuadraticNonResidue(a) { t.Fatal("element is not quadratic non residue") } } @@ -467,142 +498,277 @@ func TestFpNonResidue(t *testing.T) { } } +func TestWFp(t *testing.T) { + w := new(wfe) + a := new(fe) + fromWide(a, w) + if !a.isZero() { + t.Fatal("expect zero") + } + w[0] = r1[0] + w[1] = r1[1] + w[2] = r1[2] + w[3] = r1[3] + w[4] = r1[4] + w[5] = r1[5] + fromWide(a, w) + if !(a[0] == 1 && a[1] == 0 && a[2] == 0 && a[3] == 0 && a[4] == 0 && a[5] == 0) { + t.Fatal("expect one") + } +} + +func TestWFpAddition(t *testing.T) { + for i := 0; i < fuz; i++ { + a, _ := new(fe).rand(rand.Reader) + b, _ := new(fe).rand(rand.Reader) + w0, w1 := new(wfe), new(wfe) + c0, c1 := new(fe), new(fe) + + wmul(w0, a, b) + w1.set(w0) + wadd(w0, w0, w0) + wadd(w0, w0, w0) + wadd(w0, w0, w0) + lwadd(w1, w1, w1) + lwadd(w1, w1, w1) + lwadd(w1, w1, w1) + fromWide(c0, w0) + fromWide(c1, w1) + + if !c1.equal(c0) { + t.Fatal("addition failed") + } + + wmul(w0, a, b) + w1.set(w0) + wdouble(w0, w0) + wdouble(w0, w0) + wdouble(w0, w0) + lwdouble(w1, w1) + lwdouble(w1, w1) + lwdouble(w1, w1) + fromWide(c0, w0) + fromWide(c1, w1) + + if !c1.equal(c0) { + t.Fatal("doubling failed") + } + + wmul(w0, a, &fe{10001}) + wmul(w1, a, &fe{10000}) + w2 := new(wfe) + wsub(w2, w0, w1) + lwsub(w0, w0, w1) + fromWide(c0, w2) + fromWide(c1, w0) + + fromMont(a, a) + if !c1.equal(a) { + t.Fatal("subtraction failed") + } + if !c0.equal(a) { + t.Fatal("subtraction failed") + } + + wmul(w0, a, &fe{10001}) + wmul(w1, a, &fe{10000}) + wsub(w0, w1, w0) + fromWide(c0, w0) + + neg(a, a) + fromMont(a, a) + if !c0.equal(a) { + t.Fatal("subtraction failed") + } + + } +} + +func TestWFpMultiplication(t *testing.T) { + for i := 0; i < fuz; i++ { + a0, _ := new(fe).rand(rand.Reader) + b0, _ := new(fe).rand(rand.Reader) + a1, _ := new(fe).rand(rand.Reader) + b1, _ := new(fe).rand(rand.Reader) + w0, w1, w2, w3 := new(wfe), new(wfe), new(wfe), new(wfe) + c0, c1 := new(fe), new(fe) + r0, r1 := new(fe), new(fe) + + wmul(w0, a0, b0) + fromWide(r0, w0) + mul(r1, a0, b0) + + if !r1.equal(r0) { + t.Fatal("multiplication failed") + } + + wmul(w0, a0, b0) + wmul(w1, a1, b1) + lwadd(w0, w0, w1) + fromWide(r0, w0) + + mul(c0, a0, b0) + mul(c1, a1, b1) + add(r1, c0, c1) + + if !r1.equal(r0) { + t.Fatal("multiplication failed") + } + + wmul(w0, a0, b0) + wmul(w1, a0, b1) + wmul(w2, a1, b0) + wmul(w3, a1, b1) + lwadd(w0, w0, w1) + lwadd(w0, w0, w2) + lwadd(w0, w0, w3) + fromWide(r0, w0) + + add(c0, a0, a1) + add(c1, b0, b1) + mul(r1, c0, c1) + + if !r1.equal(r0) { + t.Fatal("multiplication failed") + } + } +} + func TestFp2Serialization(t *testing.T) { - field := newFp2() + f := newFp2() for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) - b, err := field.fromBytes(field.toBytes(a)) + b, err := f.fromBytes(f.toBytes(a)) if err != nil { t.Fatal(err) } if !a.equal(b) { - t.Fatal("bad serialization") + t.Fatal("serialization failed") } } } func TestFp2AdditionProperties(t *testing.T) { - field := newFp2() + f := newFp2() for i := 0; i < fuz; i++ { - zero := field.zero() + zero := f.zero() a, _ := new(fe2).rand(rand.Reader) b, _ := new(fe2).rand(rand.Reader) - c_1 := field.new() - c_2 := field.new() - field.add(c_1, a, zero) - if !c_1.equal(a) { + c1 := f.new() + c2 := f.new() + fp2Add(c1, a, zero) + if !c1.equal(a) { t.Fatal("a + 0 == a") } - field.sub(c_1, a, zero) - if !c_1.equal(a) { + fp2Sub(c1, a, zero) + if !c1.equal(a) { t.Fatal("a - 0 == a") } - field.double(c_1, zero) - if !c_1.equal(zero) { + fp2Double(c1, zero) + if !c1.equal(zero) { t.Fatal("2 * 0 == 0") } - field.neg(c_1, zero) - if !c_1.equal(zero) { + fp2Neg(c1, zero) + if !c1.equal(zero) { t.Fatal("-0 == 0") } - field.sub(c_1, zero, a) - field.neg(c_2, a) - if !c_1.equal(c_2) { + fp2Sub(c1, zero, a) + fp2Neg(c2, a) + if !c1.equal(c2) { t.Fatal("0-a == -a") } - field.double(c_1, a) - field.add(c_2, a, a) - if !c_1.equal(c_2) { + fp2Double(c1, a) + fp2Add(c2, a, a) + if !c1.equal(c2) { t.Fatal("2 * a == a + a") } - field.add(c_1, a, b) - field.add(c_2, b, a) - if !c_1.equal(c_2) { + fp2Add(c1, a, b) + fp2Add(c2, b, a) + if !c1.equal(c2) { t.Fatal("a + b = b + a") } - field.sub(c_1, a, b) - field.sub(c_2, b, a) - field.neg(c_2, c_2) - if !c_1.equal(c_2) { + fp2Sub(c1, a, b) + fp2Sub(c2, b, a) + fp2Neg(c2, c2) + if !c1.equal(c2) { t.Fatal("a - b = - ( b - a )") } - c_x, _ := new(fe2).rand(rand.Reader) - field.add(c_1, a, b) - field.add(c_1, c_1, c_x) - field.add(c_2, a, c_x) - field.add(c_2, c_2, b) - if !c_1.equal(c_2) { + cx, _ := new(fe2).rand(rand.Reader) + fp2Add(c1, a, b) + fp2Add(c1, c1, cx) + fp2Add(c2, a, cx) + fp2Add(c2, c2, b) + if !c1.equal(c2) { t.Fatal("(a + b) + c == (a + c ) + b") } - field.sub(c_1, a, b) - field.sub(c_1, c_1, c_x) - field.sub(c_2, a, c_x) - field.sub(c_2, c_2, b) - if !c_1.equal(c_2) { + fp2Sub(c1, a, b) + fp2Sub(c1, c1, cx) + fp2Sub(c2, a, cx) + fp2Sub(c2, c2, b) + if !c1.equal(c2) { t.Fatal("(a - b) - c == (a - c ) -b") } } } func TestFp2AdditionPropertiesAssigned(t *testing.T) { - field := newFp2() for i := 0; i < fuz; i++ { zero := new(fe2).zero() a, b := new(fe2), new(fe2) _, _ = a.rand(rand.Reader) b.set(a) - field.addAssign(a, zero) + fp2AddAssign(a, zero) if !a.equal(b) { t.Fatal("a + 0 == a") } - field.subAssign(a, zero) + fp2SubAssign(a, zero) if !a.equal(b) { t.Fatal("a - 0 == a") } a.set(zero) - field.doubleAssign(a) + fp2DoubleAssign(a) if !a.equal(zero) { t.Fatal("2 * 0 == 0") } a.set(zero) - field.subAssign(a, b) - field.neg(b, b) + fp2SubAssign(a, b) + fp2Neg(b, b) if !a.equal(b) { t.Fatal("0-a == -a") } _, _ = a.rand(rand.Reader) b.set(a) - field.doubleAssign(a) - field.addAssign(b, b) + fp2DoubleAssign(a) + fp2AddAssign(b, b) if !a.equal(b) { t.Fatal("2 * a == a + a") } _, _ = a.rand(rand.Reader) _, _ = b.rand(rand.Reader) - c_1, c_2 := new(fe2).set(a), new(fe2).set(b) - field.addAssign(c_1, b) - field.addAssign(c_2, a) - if !c_1.equal(c_2) { + c1, c2 := new(fe2).set(a), new(fe2).set(b) + fp2AddAssign(c1, b) + fp2AddAssign(c2, a) + if !c1.equal(c2) { t.Fatal("a + b = b + a") } _, _ = a.rand(rand.Reader) _, _ = b.rand(rand.Reader) - c_1.set(a) - c_2.set(b) - field.subAssign(c_1, b) - field.subAssign(c_2, a) - field.neg(c_2, c_2) - if !c_1.equal(c_2) { + c1.set(a) + c2.set(b) + fp2SubAssign(c1, b) + fp2SubAssign(c2, a) + fp2Neg(c2, c2) + if !c1.equal(c2) { t.Fatal("a - b = - ( b - a )") } _, _ = a.rand(rand.Reader) _, _ = b.rand(rand.Reader) c, _ := new(fe2).rand(rand.Reader) a0 := new(fe2).set(a) - field.addAssign(a, b) - field.addAssign(a, c) - field.addAssign(b, c) - field.addAssign(b, a0) + fp2AddAssign(a, b) + fp2AddAssign(a, c) + fp2AddAssign(b, c) + fp2AddAssign(b, a0) if !a.equal(b) { t.Fatal("(a + b) + c == (b + c) + a") } @@ -610,10 +776,10 @@ func TestFp2AdditionPropertiesAssigned(t *testing.T) { _, _ = b.rand(rand.Reader) _, _ = c.rand(rand.Reader) a0.set(a) - field.subAssign(a, b) - field.subAssign(a, c) - field.subAssign(a0, c) - field.subAssign(a0, b) + fp2SubAssign(a, b) + fp2SubAssign(a, c) + fp2SubAssign(a0, c) + fp2SubAssign(a0, b) if !a.equal(a0) { t.Fatal("(a - b) - c == (a - c) -b") } @@ -621,112 +787,132 @@ func TestFp2AdditionPropertiesAssigned(t *testing.T) { } func TestFp2LazyOperations(t *testing.T) { - field := newFp2() + f := newFp2() for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) b, _ := new(fe2).rand(rand.Reader) c, _ := new(fe2).rand(rand.Reader) c0 := new(fe2) c1 := new(fe2) - field.ladd(c0, a, b) - field.add(c1, a, b) - field.mulAssign(c0, c) - field.mulAssign(c1, c) + fp2Ladd(c0, a, b) + fp2Add(c1, a, b) + fp2LaddAssign(a, b) + f.mulAssign(c0, c) + f.mulAssign(c1, c) + f.mulAssign(a, c) if !c0.equal(c1) { - // l+ operator stands for lazy addition t.Fatal("(a + b) * c == (a l+ b) * c") } - _, _ = a.rand(rand.Reader) - b.set(a) - field.ldouble(a, a) - field.ladd(b, b, b) - if !a.equal(b) { - t.Fatal("2 l* a = a l+ a") + if !c0.equal(c1) { + t.Fatal("(a + b) * c == (a l+ b) * c") + } + } +} + +func TestFp2MulByNonResidue(t *testing.T) { + f := newFp2() + for i := 0; i < fuz; i++ { + a, _ := new(fe2).rand(rand.Reader) + r0, r1, r2, r3 := new(fe2), new(fe2), new(fe2), new(fe2) + f.mul(r1, a, nonResidue2) + mulByNonResidue(r0, a) + r2.set(a) + mulByNonResidueAssign(r2) + _fp2MulByNonResidue(r3, a) + + if !r0.equal(r1) { + t.Fatal("mul by non residue failed") + } + if !r0.equal(r2) { + t.Fatal("mul by non residue failed") + } + if !r0.equal(r3) { + t.Fatal("mul by non residue failed") } } } func TestFp2MultiplicationProperties(t *testing.T) { - field := newFp2() + f := newFp2() for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) b, _ := new(fe2).rand(rand.Reader) - zero := field.zero() - one := field.one() - c_1, c_2 := field.new(), field.new() - field.mul(c_1, a, zero) - if !c_1.equal(zero) { + zero := f.zero() + one := f.one() + c1, c2 := f.new(), f.new() + f.mul(c1, a, zero) + if !c1.equal(zero) { t.Fatal("a * 0 == 0") } - field.mul(c_1, a, one) - if !c_1.equal(a) { + f.mul(c1, a, one) + if !c1.equal(a) { t.Fatal("a * 1 == a") } - field.mul(c_1, a, b) - field.mul(c_2, b, a) - if !c_1.equal(c_2) { + f.mul(c1, a, b) + f.mul(c2, b, a) + if !c1.equal(c2) { t.Fatal("a * b == b * a") } - c_x, _ := new(fe2).rand(rand.Reader) - field.mul(c_1, a, b) - field.mul(c_1, c_1, c_x) - field.mul(c_2, c_x, b) - field.mul(c_2, c_2, a) - if !c_1.equal(c_2) { + cx, _ := new(fe2).rand(rand.Reader) + f.mul(c1, a, b) + f.mul(c1, c1, cx) + f.mul(c2, cx, b) + f.mul(c2, c2, a) + if !c1.equal(c2) { t.Fatal("(a * b) * c == (a * c) * b") } - field.square(a, zero) + f.square(a, zero) if !a.equal(zero) { t.Fatal("0^2 == 0") } - field.square(a, one) + f.square(a, one) if !a.equal(one) { t.Fatal("1^2 == 1") } _, _ = a.rand(rand.Reader) - field.square(c_1, a) - field.mul(c_2, a, a) - if !c_2.equal(c_1) { + f.square(c1, a) + f.mul(c2, a, a) + if !c2.equal(c1) { t.Fatal("a^2 == a*a") } } } func TestFp2MultiplicationPropertiesAssigned(t *testing.T) { - field := newFp2() + f := newFp2() for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) zero, one := new(fe2).zero(), new(fe2).one() - field.mulAssign(a, zero) + f.mulAssign(a, zero) if !a.equal(zero) { t.Fatal("a * 0 == 0") } _, _ = a.rand(rand.Reader) a0 := new(fe2).set(a) - field.mulAssign(a, one) + f.mulAssign(a, one) if !a.equal(a0) { t.Fatal("a * 1 == a") } _, _ = a.rand(rand.Reader) b, _ := new(fe2).rand(rand.Reader) a0.set(a) - field.mulAssign(a, b) - field.mulAssign(b, a0) + f.mulAssign(a, b) + f.mulAssign(b, a0) if !a.equal(b) { t.Fatal("a * b == b * a") } c, _ := new(fe2).rand(rand.Reader) a0.set(a) - field.mulAssign(a, b) - field.mulAssign(a, c) - field.mulAssign(a0, c) - field.mulAssign(a0, b) + f.mulAssign(a, b) + f.mulAssign(a, c) + f.mulAssign(a0, c) + f.mulAssign(a0, b) if !a.equal(a0) { t.Fatal("(a * b) * c == (a * c) * b") } a0.set(a) - field.squareAssign(a) - field.mulAssign(a0, a0) + f.squareAssign(a) + f.mulAssign(a0, a0) if !a.equal(a0) { t.Fatal("a^2 == a*a") } @@ -734,23 +920,23 @@ func TestFp2MultiplicationPropertiesAssigned(t *testing.T) { } func TestFp2Exponentiation(t *testing.T) { - field := newFp2() + f := newFp2() for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) - u := field.new() - field.exp(u, a, big.NewInt(0)) - if !u.equal(field.one()) { + u := f.new() + f.exp(u, a, big.NewInt(0)) + if !u.equal(f.one()) { t.Fatal("a^0 == 1") } - field.exp(u, a, big.NewInt(1)) + f.exp(u, a, big.NewInt(1)) if !u.equal(a) { t.Fatal("a^1 == a") } - v := field.new() - field.mul(u, a, a) - field.mul(u, u, u) - field.mul(u, u, u) - field.exp(v, a, big.NewInt(8)) + v := f.new() + f.mul(u, a, a) + f.mul(u, u, u) + f.mul(u, u, u) + f.exp(v, a, big.NewInt(8)) if !u.equal(v) { t.Fatal("((a^2)^2)^2 == a^8") } @@ -758,90 +944,101 @@ func TestFp2Exponentiation(t *testing.T) { } func TestFp2Inversion(t *testing.T) { - field := newFp2() - u := field.new() - zero := field.zero() - one := field.one() - field.inverse(u, zero) + f := newFp2() + u := f.new() + zero := f.zero() + one := f.one() + f.inverse(u, zero) if !u.equal(zero) { t.Fatal("(0 ^ -1) == 0)") } - field.inverse(u, one) + f.inverse(u, one) if !u.equal(one) { t.Fatal("(1 ^ -1) == 1)") } for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) - field.inverse(u, a) - field.mul(u, u, a) + f.inverse(u, a) + f.mul(u, u, a) if !u.equal(one) { t.Fatal("(r * a) * r * (a ^ -1) == r)") } } } -func TestFp2SquareRoot(t *testing.T) { - field := newFp2() - for z := 0; z < 1000; z++ { - zi := new(fe) - sub(zi, &modulus, &fe{uint64(z * z)}) - // r = (-z*z, 0) - r := &fe2{*zi, fe{0}} - toMont(&r[0], &r[0]) - toMont(&r[1], &r[1]) - c := field.new() - // sqrt((-z*z, 0)) = (0, z) - if !field.sqrt(c, r) { - t.Fatal("z*z does have a square root") +func TestFp2BatchInversion(t *testing.T) { + f := newFp2() + n := 20 + for i := 0; i < n; i++ { + e0 := make([]fe2, n) + e1 := make([]fe2, n) + for j := 0; j < n; j++ { + if j != i { + e, err := new(fe2).rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + e0[j].set(e) + } + f.inverse(&e1[j], &e0[j]) } - e := &fe2{fe{uint64(0)}, fe{uint64(z)}} - toMont(&e[0], &e[0]) - toMont(&e[1], &e[1]) - field.square(e, e) - field.square(c, c) - if !e.equal(c) { - t.Fatal("square root failed") + f.inverseBatch(e0) + for j := 0; j < n; j++ { + if !e0[j].equal(&e1[j]) { + t.Fatal("batch inversion failed") + } } } - if field.sqrt(field.new(), nonResidue2) { +} + +func TestFp2SquareRoot(t *testing.T) { + e := newFp2() + if e.sqrtBLST(e.new(), nonResidue2) { t.Fatal("non residue cannot have a sqrt") } for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) - aa, rr, r := field.new(), field.new(), field.new() - field.square(aa, a) - if !field.sqrt(r, aa) { - t.Fatal("bad sqrt 1") + r0, r1 := new(fe2), new(fe2) + d0 := e.sqrt(r0, a) + d1 := e.sqrtBLST(r1, a) + if d0 != d1 { + t.Fatal("sqrt decision failed") } - field.square(rr, r) - if !rr.equal(aa) { - t.Fatal("bad sqrt 2") + if d0 { + e.square(r0, r0) + e.square(r1, r1) + if !r0.equal(r1) { + t.Fatal("sqrt failed") + } + if !r0.equal(a) { + t.Fatal("sqrt failed") + } } } } func TestFp2NonResidue(t *testing.T) { - field := newFp2() - if !field.isQuadraticNonResidue(nonResidue2) { + f := newFp2() + if !f.isQuadraticNonResidue(nonResidue2) { t.Fatal("element is quadratic non residue, 1") } - if field.isQuadraticNonResidue(new(fe2).one()) { + if f.isQuadraticNonResidue(new(fe2).one()) { t.Fatal("one is not quadratic non residue") } - if !field.isQuadraticNonResidue(new(fe2).zero()) { + if !f.isQuadraticNonResidue(new(fe2).zero()) { t.Fatal("should accept zero as quadratic non residue") } for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) - field.squareAssign(a) - if field.isQuadraticNonResidue(new(fe2).one()) { + f.squareAssign(a) + if f.isQuadraticNonResidue(a) { t.Fatal("element is not quadratic non residue") } } for i := 0; i < fuz; i++ { a, _ := new(fe2).rand(rand.Reader) - if !field.sqrt(new(fe2), a) { - if !field.isQuadraticNonResidue(a) { + if !f.sqrt(new(fe2), a) { + if !f.isQuadraticNonResidue(a) { t.Fatal("element is quadratic non residue, 2", i) } } else { @@ -850,142 +1047,289 @@ func TestFp2NonResidue(t *testing.T) { } } +func TestWFp2Addition(t *testing.T) { + for i := 0; i < fuz; i++ { + r0, _ := new(fe2).rand(rand.Reader) + r1, _ := new(fe2).rand(rand.Reader) + r2, _ := new(fe2).rand(rand.Reader) + rw0, rw1, w0, w1 := new(wfe2), new(wfe2), new(wfe2), new(wfe2) + + wfp2Mul(w0, r0, r1) + wfp2Mul(w1, r0, r2) + + _wfp2Add(rw0, w0, w1) + wfp2Add(rw1, w0, w1) + if !rw0.equal(rw1) { + t.Fatal("add failed") + } + rw1.set(w0) + wfp2AddAssign(rw1, w1) + if !rw0.equal(rw1) { + t.Fatal("assigned add failed") + } + + _wfp2AddMixed(rw0, w0, w1) + wfp2AddMixed(rw1, w0, w1) + if !rw0.equal(rw1) { + t.Fatal("add mixed failed") + } + rw1.set(w0) + wfp2AddMixedAssign(rw1, w1) + if !rw0.equal(rw1) { + t.Fatal("assigned mixed add failed") + } + + _wfp2Ladd(rw0, w0, w1) + wfp2Ladd(rw1, w0, w1) + if !rw0.equal(rw1) { + t.Fatal("lazy add failed") + } + rw1.set(w0) + wfp2LaddAssign(rw1, w1) + if !rw0.equal(rw1) { + t.Fatal("assigned lazy add failed") + } + + _wfp2Sub(rw0, w0, w1) + wfp2Sub(rw1, w0, w1) + if !rw0.equal(rw1) { + t.Fatal("sub failed") + } + rw1.set(w0) + wfp2SubAssign(rw1, w1) + if !rw0.equal(rw1) { + t.Fatal("assigned sub failed") + } + + _wfp2SubMixed(rw0, w0, w1) + wfp2SubMixed(rw1, w0, w1) + if !rw0.equal(rw1) { + t.Fatal("sub mixed failed") + } + rw1.set(w0) + wfp2SubMixedAssign(rw1, w1) + if !rw0.equal(rw1) { + t.Fatal("assigned sub mixed failed") + } + + _wfp2Double(rw0, w0) + wfp2Double(rw1, w0) + if !rw0.equal(rw1) { + t.Fatal("doubling failed") + } + rw1.set(w0) + wfp2DoubleAssign(rw1) + if !rw0.equal(rw1) { + t.Fatal("assigned doubling failed") + } + + } +} + +func TestFp2MultiplicationCross(t *testing.T) { + f := newFp2() + a, b, c0, c1 := new(fe2), new(fe2), new(fe2), new(fe2) + w0, w1 := new(wfe2), new(wfe2) + for i := 0; i < fuz; i++ { + _, _ = a.rand(rand.Reader) + _, _ = b.rand(rand.Reader) + _wfp2Mul(w0, a, b) + wfp2Mul(w1, a, b) + if !w0.equal(w1) { + t.Fatal("multiplication failed") + } + c0.fromWide(w0) + f.mul(c1, a, b) + if !c0.equal(c1) { + t.Fatal("multiplication failed") + } + } +} + +func TestFp2SquareCross(t *testing.T) { + f := newFp2() + a, c0, c1 := new(fe2), new(fe2), new(fe2) + w0, w1 := new(wfe2), new(wfe2) + for i := 0; i < fuz; i++ { + _, _ = a.rand(rand.Reader) + _wfp2Square(w0, a) + wfp2Square(w1, a) + if !w0.equal(w1) { + t.Fatal("squaring failed") + } + c0.fromWide(w0) + f.square(c1, a) + if !c0.equal(c1) { + t.Fatal("squaring failed") + } + } +} + +func TestWFp2MulByNonResidue(t *testing.T) { + f := newFp2() + a, b, c0, c1 := new(fe2), new(fe2), new(fe2), new(fe2) + w0, w1, w2, w3 := new(wfe2), new(wfe2), new(wfe2), new(wfe2) + for i := 0; i < fuz; i++ { + _, _ = a.rand(rand.Reader) + _, _ = b.rand(rand.Reader) + r := new(fe2) + + f.mul(r, a, b) + wfp2Mul(w0, a, b) + + mulByNonResidue(c0, r) + wfp2MulByNonResidue(w1, w0) + w2.set(w0) + wfp2MulByNonResidueAssign(w2) + _wfp2MulByNonResidue(w3, w0) + if !w1.equal(w2) { + t.Fatal("mul by non residue failed") + } + if !w1.equal(w3) { + t.Fatal("mul by non residue failed") + } + c1.fromWide(w1) + if !c0.equal(c1) { + t.Fatal("mul by non residue failed") + } + } +} + func TestFp6Serialization(t *testing.T) { - field := newFp6(nil) + f := newFp6(nil) for i := 0; i < fuz; i++ { a, _ := new(fe6).rand(rand.Reader) - b, err := field.fromBytes(field.toBytes(a)) + b, err := f.fromBytes(f.toBytes(a)) if err != nil { t.Fatal(err) } if !a.equal(b) { - t.Fatal("bad serialization") + t.Fatal("serialization") } } } func TestFp6AdditionProperties(t *testing.T) { - field := newFp6(nil) + f := newFp6(nil) for i := 0; i < fuz; i++ { - zero := field.zero() + zero := f.zero() a, _ := new(fe6).rand(rand.Reader) b, _ := new(fe6).rand(rand.Reader) - c_1 := field.new() - c_2 := field.new() - field.add(c_1, a, zero) - if !c_1.equal(a) { + c1 := f.new() + c2 := f.new() + fp6Add(c1, a, zero) + if !c1.equal(a) { t.Fatal("a + 0 == a") } - field.sub(c_1, a, zero) - if !c_1.equal(a) { + fp6Sub(c1, a, zero) + if !c1.equal(a) { t.Fatal("a - 0 == a") } - field.double(c_1, zero) - if !c_1.equal(zero) { + fp6Double(c1, zero) + if !c1.equal(zero) { t.Fatal("2 * 0 == 0") } - field.neg(c_1, zero) - if !c_1.equal(zero) { + fp6Neg(c1, zero) + if !c1.equal(zero) { t.Fatal("-0 == 0") } - field.sub(c_1, zero, a) - field.neg(c_2, a) - if !c_1.equal(c_2) { + fp6Sub(c1, zero, a) + fp6Neg(c2, a) + if !c1.equal(c2) { t.Fatal("0-a == -a") } - field.double(c_1, a) - field.add(c_2, a, a) - if !c_1.equal(c_2) { + fp6Double(c1, a) + fp6Add(c2, a, a) + if !c1.equal(c2) { t.Fatal("2 * a == a + a") } - field.add(c_1, a, b) - field.add(c_2, b, a) - if !c_1.equal(c_2) { + fp6Add(c1, a, b) + fp6Add(c2, b, a) + if !c1.equal(c2) { t.Fatal("a + b = b + a") } - field.sub(c_1, a, b) - field.sub(c_2, b, a) - field.neg(c_2, c_2) - if !c_1.equal(c_2) { + fp6Sub(c1, a, b) + fp6Sub(c2, b, a) + fp6Neg(c2, c2) + if !c1.equal(c2) { t.Fatal("a - b = - ( b - a )") } - c_x, _ := new(fe6).rand(rand.Reader) - field.add(c_1, a, b) - field.add(c_1, c_1, c_x) - field.add(c_2, a, c_x) - field.add(c_2, c_2, b) - if !c_1.equal(c_2) { + cx, _ := new(fe6).rand(rand.Reader) + fp6Add(c1, a, b) + fp6Add(c1, c1, cx) + fp6Add(c2, a, cx) + fp6Add(c2, c2, b) + if !c1.equal(c2) { t.Fatal("(a + b) + c == (a + c ) + b") } - field.sub(c_1, a, b) - field.sub(c_1, c_1, c_x) - field.sub(c_2, a, c_x) - field.sub(c_2, c_2, b) - if !c_1.equal(c_2) { + fp6Sub(c1, a, b) + fp6Sub(c1, c1, cx) + fp6Sub(c2, a, cx) + fp6Sub(c2, c2, b) + if !c1.equal(c2) { t.Fatal("(a - b) - c == (a - c ) -b") } } } func TestFp6AdditionPropertiesAssigned(t *testing.T) { - field := newFp6(nil) for i := 0; i < fuz; i++ { zero := new(fe6).zero() a, b := new(fe6), new(fe6) _, _ = a.rand(rand.Reader) b.set(a) - field.addAssign(a, zero) + fp6AddAssign(a, zero) if !a.equal(b) { t.Fatal("a + 0 == a") } - field.subAssign(a, zero) + fp6SubAssign(a, zero) if !a.equal(b) { t.Fatal("a - 0 == a") } a.set(zero) - field.doubleAssign(a) + fp6DoubleAssign(a) if !a.equal(zero) { t.Fatal("2 * 0 == 0") } a.set(zero) - field.subAssign(a, b) - field.neg(b, b) + fp6SubAssign(a, b) + fp6Neg(b, b) if !a.equal(b) { t.Fatal("0-a == -a") } _, _ = a.rand(rand.Reader) b.set(a) - field.doubleAssign(a) - field.addAssign(b, b) + fp6DoubleAssign(a) + fp6AddAssign(b, b) if !a.equal(b) { t.Fatal("2 * a == a + a") } _, _ = a.rand(rand.Reader) _, _ = b.rand(rand.Reader) - c_1, c_2 := new(fe6).set(a), new(fe6).set(b) - field.addAssign(c_1, b) - field.addAssign(c_2, a) - if !c_1.equal(c_2) { + c1, c2 := new(fe6).set(a), new(fe6).set(b) + fp6AddAssign(c1, b) + fp6AddAssign(c2, a) + if !c1.equal(c2) { t.Fatal("a + b = b + a") } _, _ = a.rand(rand.Reader) _, _ = b.rand(rand.Reader) - c_1.set(a) - c_2.set(b) - field.subAssign(c_1, b) - field.subAssign(c_2, a) - field.neg(c_2, c_2) - if !c_1.equal(c_2) { + c1.set(a) + c2.set(b) + fp6SubAssign(c1, b) + fp6SubAssign(c2, a) + fp6Neg(c2, c2) + if !c1.equal(c2) { t.Fatal("a - b = - ( b - a )") } _, _ = a.rand(rand.Reader) _, _ = b.rand(rand.Reader) c, _ := new(fe6).rand(rand.Reader) a0 := new(fe6).set(a) - field.addAssign(a, b) - field.addAssign(a, c) - field.addAssign(b, c) - field.addAssign(b, a0) + fp6AddAssign(a, b) + fp6AddAssign(a, c) + fp6AddAssign(b, c) + fp6AddAssign(b, a0) if !a.equal(b) { t.Fatal("(a + b) + c == (b + c) + a") } @@ -993,119 +1337,140 @@ func TestFp6AdditionPropertiesAssigned(t *testing.T) { _, _ = b.rand(rand.Reader) _, _ = c.rand(rand.Reader) a0.set(a) - field.subAssign(a, b) - field.subAssign(a, c) - field.subAssign(a0, c) - field.subAssign(a0, b) + fp6SubAssign(a, b) + fp6SubAssign(a, c) + fp6SubAssign(a0, c) + fp6SubAssign(a0, b) if !a.equal(a0) { t.Fatal("(a - b) - c == (a - c) -b") } } } +func TestFp6LazyOperations(t *testing.T) { + f := newFp6(nil) + for i := 0; i < fuz; i++ { + a, _ := new(fe6).rand(rand.Reader) + b, _ := new(fe6).rand(rand.Reader) + c, _ := new(fe6).rand(rand.Reader) + c0 := new(fe6) + c1 := new(fe6) + fp6Ladd(c0, a, b) + fp6Add(c1, a, b) + f.mulAssign(c0, c) + f.mulAssign(c1, c) + if !c0.equal(c1) { + t.Fatal("(a + b) * c == (a l+ b) * c") + } + if !c0.equal(c1) { + t.Fatal("(a + b) * c == (a l+ b) * c") + } + } +} + func TestFp6SparseMultiplication(t *testing.T) { fp6 := newFp6(nil) var a, b, u *fe6 - for j := 0; j < fuz; j++ { + for i := 0; i < fuz; i++ { a, _ = new(fe6).rand(rand.Reader) b, _ = new(fe6).rand(rand.Reader) u, _ = new(fe6).rand(rand.Reader) b[2].zero() fp6.mul(u, a, b) - fp6.mulBy01(a, a, &b[0], &b[1]) + fp6._mul01(a, a, &b[0], &b[1]) if !a.equal(u) { - t.Fatal("bad mul by 01") + t.Fatal("mul by 01") } } - for j := 0; j < fuz; j++ { + for i := 0; i < fuz; i++ { a, _ = new(fe6).rand(rand.Reader) b, _ = new(fe6).rand(rand.Reader) u, _ = new(fe6).rand(rand.Reader) b[2].zero() b[0].zero() fp6.mul(u, a, b) - fp6.mulBy1(a, a, &b[1]) + fp6._mul1(a, a, &b[1]) if !a.equal(u) { - t.Fatal("bad mul by 1") + t.Fatal("mul by 1") } } } func TestFp6MultiplicationProperties(t *testing.T) { - field := newFp6(nil) + f := newFp6(nil) for i := 0; i < fuz; i++ { a, _ := new(fe6).rand(rand.Reader) b, _ := new(fe6).rand(rand.Reader) - zero := field.zero() - one := field.one() - c_1, c_2 := field.new(), field.new() - field.mul(c_1, a, zero) - if !c_1.equal(zero) { + zero := f.zero() + one := f.one() + c1, c2 := f.new(), f.new() + f.mul(c1, a, zero) + if !c1.equal(zero) { t.Fatal("a * 0 == 0") } - field.mul(c_1, a, one) - if !c_1.equal(a) { + f.mul(c1, a, one) + if !c1.equal(a) { t.Fatal("a * 1 == a") } - field.mul(c_1, a, b) - field.mul(c_2, b, a) - if !c_1.equal(c_2) { + f.mul(c1, a, b) + f.mul(c2, b, a) + if !c1.equal(c2) { t.Fatal("a * b == b * a") } - c_x, _ := new(fe6).rand(rand.Reader) - field.mul(c_1, a, b) - field.mul(c_1, c_1, c_x) - field.mul(c_2, c_x, b) - field.mul(c_2, c_2, a) - if !c_1.equal(c_2) { + cx, _ := new(fe6).rand(rand.Reader) + f.mul(c1, a, b) + f.mul(c1, c1, cx) + f.mul(c2, cx, b) + f.mul(c2, c2, a) + if !c1.equal(c2) { t.Fatal("(a * b) * c == (a * c) * b") } - field.square(a, zero) + f.square(a, zero) if !a.equal(zero) { t.Fatal("0^2 == 0") } - field.square(a, one) + f.square(a, one) if !a.equal(one) { t.Fatal("1^2 == 1") } _, _ = a.rand(rand.Reader) - field.square(c_1, a) - field.mul(c_2, a, a) - if !c_2.equal(c_1) { + f.square(c1, a) + f.mul(c2, a, a) + if !c2.equal(c1) { t.Fatal("a^2 == a*a") } } } func TestFp6MultiplicationPropertiesAssigned(t *testing.T) { - field := newFp6(nil) + f := newFp6(nil) for i := 0; i < fuz; i++ { a, _ := new(fe6).rand(rand.Reader) zero, one := new(fe6).zero(), new(fe6).one() - field.mulAssign(a, zero) + f.mulAssign(a, zero) if !a.equal(zero) { t.Fatal("a * 0 == 0") } _, _ = a.rand(rand.Reader) a0 := new(fe6).set(a) - field.mulAssign(a, one) + f.mulAssign(a, one) if !a.equal(a0) { t.Fatal("a * 1 == a") } _, _ = a.rand(rand.Reader) b, _ := new(fe6).rand(rand.Reader) a0.set(a) - field.mulAssign(a, b) - field.mulAssign(b, a0) + f.mulAssign(a, b) + f.mulAssign(b, a0) if !a.equal(b) { t.Fatal("a * b == b * a") } c, _ := new(fe6).rand(rand.Reader) a0.set(a) - field.mulAssign(a, b) - field.mulAssign(a, c) - field.mulAssign(a0, c) - field.mulAssign(a0, b) + f.mulAssign(a, b) + f.mulAssign(a, c) + f.mulAssign(a0, c) + f.mulAssign(a0, b) if !a.equal(a0) { t.Fatal("(a * b) * c == (a * c) * b") } @@ -1113,204 +1478,302 @@ func TestFp6MultiplicationPropertiesAssigned(t *testing.T) { } func TestFp6Exponentiation(t *testing.T) { - field := newFp6(nil) + f := newFp6(nil) for i := 0; i < fuz; i++ { a, _ := new(fe6).rand(rand.Reader) - u := field.new() - field.exp(u, a, big.NewInt(0)) - if !u.equal(field.one()) { + u := f.new() + f.exp(u, a, big.NewInt(0)) + if !u.equal(f.one()) { t.Fatal("a^0 == 1") } - field.exp(u, a, big.NewInt(1)) + f.exp(u, a, big.NewInt(1)) if !u.equal(a) { t.Fatal("a^1 == a") } - v := field.new() - field.mul(u, a, a) - field.mul(u, u, u) - field.mul(u, u, u) - field.exp(v, a, big.NewInt(8)) + v := f.new() + f.exp(v, a, big.NewInt(8)) + f.square(u, a) + f.square(u, u) + f.square(u, u) if !u.equal(v) { - t.Fatal("((a^2)^2)^2 == a^8") + t.Fatal("((a^2)^2)^2 == a^8", i) } } } func TestFp6Inversion(t *testing.T) { - field := newFp6(nil) + f := newFp6(nil) for i := 0; i < fuz; i++ { - u := field.new() - zero := field.zero() - one := field.one() - field.inverse(u, zero) + u := f.new() + zero := f.zero() + one := f.one() + f.inverse(u, zero) if !u.equal(zero) { t.Fatal("(0^-1) == 0)") } - field.inverse(u, one) + f.inverse(u, one) if !u.equal(one) { t.Fatal("(1^-1) == 1)") } a, _ := new(fe6).rand(rand.Reader) - field.inverse(u, a) - field.mul(u, u, a) + f.inverse(u, a) + f.mul(u, u, a) if !u.equal(one) { t.Fatal("(r*a) * r*(a^-1) == r)") } } } +func TestFp6MultiplicationCross(t *testing.T) { + f := newFp6(nil) + a, b, c0, c1, c2, c3 := new(fe6), new(fe6), new(fe6), new(fe6), new(fe6), new(fe6) + w0 := new(wfe6) + for i := 0; i < fuz; i++ { + + _, _ = a.rand(rand.Reader) + _, _ = b.rand(rand.Reader) + f.wmul(w0, a, b) + c0.fromWide(w0) + f.mul(c1, a, b) + f._mul(c2, a, b) + c3.set(a) + f.mulAssign(c3, b) + if !c0.equal(c1) { + t.Fatal("multiplication failed") + } + if !c0.equal(c2) { + t.Fatal("multiplication failed") + } + if !c0.equal(c3) { + t.Fatal("multiplication failed") + } + + } +} + +func TestFp6SquareCross(t *testing.T) { + f := newFp6(nil) + a, c0, c1, c2 := new(fe6), new(fe6), new(fe6), new(fe6) + w0 := new(wfe6) + for i := 0; i < fuz; i++ { + _, _ = a.rand(rand.Reader) + f.wsquare(w0, a) + c0.fromWide(w0) + f.square(c1, a) + f._square(c2, a) + + if !c0.equal(c2) { + t.Fatal("squaring failed") + } + if !c0.equal(c1) { + t.Fatal("squaring failed") + } + } +} + +func TestFp6SparseMultiplicationCross(t *testing.T) { + f := newFp6(nil) + a, c0, c1, c2 := new(fe6), new(fe6), new(fe6), new(fe6) + w0 := new(wfe6) + for i := 0; i < fuz; i++ { + // mul01 + { + _, _ = a.rand(rand.Reader) + b0, _ := new(fe2).rand(rand.Reader) + b1, _ := new(fe2).rand(rand.Reader) + b := new(fe6) + b[0].set(b0) + b[1].set(b1) + + f.wmul01(w0, a, b0, b1) + c0.fromWide(w0) + + f._mul01(c1, a, b0, b1) + f._mul(c2, a, b) + + if !c2.equal(c1) { + t.Fatal("sparse multiplication 01 failed") + } + + if !c0.equal(c1) { + t.Fatal("sparse multiplication 01 failed") + } + + } + // mul0 + { + _, _ = a.rand(rand.Reader) + b1, _ := new(fe2).rand(rand.Reader) + b := new(fe6) + b[1].set(b1) + + f.wmul1(w0, a, b1) + c0.fromWide(w0) + f._mul1(c1, a, b1) + f._mul(c2, a, b) + + if !c2.equal(c0) { + t.Fatal("sparse multiplication 0 failed") + } + if !c2.equal(c1) { + t.Fatal("sparse multiplication 0 failed") + } + } + } +} + func TestFp12Serialization(t *testing.T) { - field := newFp12(nil) + f := newFp12(nil) for i := 0; i < fuz; i++ { a, _ := new(fe12).rand(rand.Reader) - b, err := field.fromBytes(field.toBytes(a)) + b, err := f.fromBytes(f.toBytes(a)) if err != nil { t.Fatal(err) } if !a.equal(b) { - t.Fatal("bad serialization") + t.Fatal("serialization") } } } func TestFp12AdditionProperties(t *testing.T) { - field := newFp12(nil) + f := newFp12(nil) for i := 0; i < fuz; i++ { - zero := field.zero() + zero := f.zero() a, _ := new(fe12).rand(rand.Reader) b, _ := new(fe12).rand(rand.Reader) - c_1 := field.new() - c_2 := field.new() - field.add(c_1, a, zero) - if !c_1.equal(a) { + c1 := f.new() + c2 := f.new() + fp12Add(c1, a, zero) + if !c1.equal(a) { t.Fatal("a + 0 == a") } - field.sub(c_1, a, zero) - if !c_1.equal(a) { + fp12Sub(c1, a, zero) + if !c1.equal(a) { t.Fatal("a - 0 == a") } - field.double(c_1, zero) - if !c_1.equal(zero) { + fp12Double(c1, zero) + if !c1.equal(zero) { t.Fatal("2 * 0 == 0") } - field.neg(c_1, zero) - if !c_1.equal(zero) { + fp12Neg(c1, zero) + if !c1.equal(zero) { t.Fatal("-0 == 0") } - field.sub(c_1, zero, a) - field.neg(c_2, a) - if !c_1.equal(c_2) { + fp12Sub(c1, zero, a) + fp12Neg(c2, a) + if !c1.equal(c2) { t.Fatal("0-a == -a") } - field.double(c_1, a) - field.add(c_2, a, a) - if !c_1.equal(c_2) { + fp12Double(c1, a) + fp12Add(c2, a, a) + if !c1.equal(c2) { t.Fatal("2 * a == a + a") } - field.add(c_1, a, b) - field.add(c_2, b, a) - if !c_1.equal(c_2) { + fp12Add(c1, a, b) + fp12Add(c2, b, a) + if !c1.equal(c2) { t.Fatal("a + b = b + a") } - field.sub(c_1, a, b) - field.sub(c_2, b, a) - field.neg(c_2, c_2) - if !c_1.equal(c_2) { + fp12Sub(c1, a, b) + fp12Sub(c2, b, a) + fp12Neg(c2, c2) + if !c1.equal(c2) { t.Fatal("a - b = - ( b - a )") } - c_x, _ := new(fe12).rand(rand.Reader) - field.add(c_1, a, b) - field.add(c_1, c_1, c_x) - field.add(c_2, a, c_x) - field.add(c_2, c_2, b) - if !c_1.equal(c_2) { + cx, _ := new(fe12).rand(rand.Reader) + fp12Add(c1, a, b) + fp12Add(c1, c1, cx) + fp12Add(c2, a, cx) + fp12Add(c2, c2, b) + if !c1.equal(c2) { t.Fatal("(a + b) + c == (a + c ) + b") } - field.sub(c_1, a, b) - field.sub(c_1, c_1, c_x) - field.sub(c_2, a, c_x) - field.sub(c_2, c_2, b) - if !c_1.equal(c_2) { + fp12Sub(c1, a, b) + fp12Sub(c1, c1, cx) + fp12Sub(c2, a, cx) + fp12Sub(c2, c2, b) + if !c1.equal(c2) { t.Fatal("(a - b) - c == (a - c ) -b") } } } func TestFp12MultiplicationProperties(t *testing.T) { - field := newFp12(nil) + f := newFp12(nil) for i := 0; i < fuz; i++ { a, _ := new(fe12).rand(rand.Reader) b, _ := new(fe12).rand(rand.Reader) - zero := field.zero() - one := field.one() - c_1, c_2 := field.new(), field.new() - field.mul(c_1, a, zero) - if !c_1.equal(zero) { + zero := f.zero() + one := f.one() + c1, c2 := f.new(), f.new() + f.mul(c1, a, zero) + if !c1.equal(zero) { t.Fatal("a * 0 == 0") } - field.mul(c_1, a, one) - if !c_1.equal(a) { + f.mul(c1, a, one) + if !c1.equal(a) { t.Fatal("a * 1 == a") } - field.mul(c_1, a, b) - field.mul(c_2, b, a) - if !c_1.equal(c_2) { + f.mul(c1, a, b) + f.mul(c2, b, a) + if !c1.equal(c2) { t.Fatal("a * b == b * a") } - c_x, _ := new(fe12).rand(rand.Reader) - field.mul(c_1, a, b) - field.mul(c_1, c_1, c_x) - field.mul(c_2, c_x, b) - field.mul(c_2, c_2, a) - if !c_1.equal(c_2) { + cx, _ := new(fe12).rand(rand.Reader) + f.mul(c1, a, b) + f.mul(c1, c1, cx) + f.mul(c2, cx, b) + f.mul(c2, c2, a) + if !c1.equal(c2) { t.Fatal("(a * b) * c == (a * c) * b") } - field.square(a, zero) + f.square(a, zero) if !a.equal(zero) { t.Fatal("0^2 == 0") } - field.square(a, one) + f.square(a, one) if !a.equal(one) { t.Fatal("1^2 == 1") } _, _ = a.rand(rand.Reader) - field.square(c_1, a) - field.mul(c_2, a, a) - if !c_2.equal(c_1) { + f.square(c1, a) + f.mul(c2, a, a) + if !c2.equal(c1) { t.Fatal("a^2 == a*a") } } } func TestFp12MultiplicationPropertiesAssigned(t *testing.T) { - field := newFp12(nil) + f := newFp12(nil) + zero, one := new(fe12).zero(), new(fe12).one() for i := 0; i < fuz; i++ { a, _ := new(fe12).rand(rand.Reader) - zero, one := new(fe12).zero(), new(fe12).one() - field.mulAssign(a, zero) + f.mulAssign(a, zero) if !a.equal(zero) { t.Fatal("a * 0 == 0") } _, _ = a.rand(rand.Reader) a0 := new(fe12).set(a) - field.mulAssign(a, one) + f.mulAssign(a, one) if !a.equal(a0) { t.Fatal("a * 1 == a") } _, _ = a.rand(rand.Reader) b, _ := new(fe12).rand(rand.Reader) a0.set(a) - field.mulAssign(a, b) - field.mulAssign(b, a0) + f.mulAssign(a, b) + f.mulAssign(b, a0) if !a.equal(b) { t.Fatal("a * b == b * a") } c, _ := new(fe12).rand(rand.Reader) a0.set(a) - field.mulAssign(a, b) - field.mulAssign(a, c) - field.mulAssign(a0, c) - field.mulAssign(a0, b) + f.mul(a, a, b) + f.mul(a, a, c) + f.mul(a0, a0, c) + f.mul(a0, a0, b) if !a.equal(a0) { t.Fatal("(a * b) * c == (a * c) * b") } @@ -1328,31 +1791,31 @@ func TestFp12SparseMultiplication(t *testing.T) { b[1][0].zero() b[1][2].zero() fp12.mul(u, a, b) - fp12.mulBy014Assign(a, &b[0][0], &b[0][1], &b[1][1]) + fp12.mul014(a, &b[0][0], &b[0][1], &b[1][1]) if !a.equal(u) { - t.Fatal("bad mul by 01") + t.Fatal("mul by 01") } } } func TestFp12Exponentiation(t *testing.T) { - field := newFp12(nil) + f := newFp12(nil) for i := 0; i < fuz; i++ { a, _ := new(fe12).rand(rand.Reader) - u := field.new() - field.exp(u, a, big.NewInt(0)) - if !u.equal(field.one()) { + u := f.new() + f.exp(u, a, big.NewInt(0)) + if !u.equal(f.one()) { t.Fatal("a^0 == 1") } - field.exp(u, a, big.NewInt(1)) + f.exp(u, a, big.NewInt(1)) if !u.equal(a) { t.Fatal("a^1 == a") } - v := field.new() - field.mul(u, a, a) - field.mul(u, u, u) - field.mul(u, u, u) - field.exp(v, a, big.NewInt(8)) + v := f.new() + f.mul(u, a, a) + f.mul(u, u, u) + f.mul(u, u, u) + f.exp(v, a, big.NewInt(8)) if !u.equal(v) { t.Fatal("((a^2)^2)^2 == a^8") } @@ -1360,52 +1823,416 @@ func TestFp12Exponentiation(t *testing.T) { } func TestFp12Inversion(t *testing.T) { - field := newFp12(nil) + f := newFp12(nil) for i := 0; i < fuz; i++ { - u := field.new() - zero := field.zero() - one := field.one() - field.inverse(u, zero) + u := f.new() + zero := f.zero() + one := f.one() + f.inverse(u, zero) if !u.equal(zero) { t.Fatal("(0^-1) == 0)") } - field.inverse(u, one) + f.inverse(u, one) if !u.equal(one) { t.Fatal("(1^-1) == 1)") } a, _ := new(fe12).rand(rand.Reader) - field.inverse(u, a) - field.mul(u, u, a) + f.inverse(u, a) + f.mul(u, u, a) if !u.equal(one) { t.Fatal("(r*a) * r*(a^-1) == r)") } } } -func BenchmarkMultiplication(t *testing.B) { +func TestFrobeniusMapping2(t *testing.T) { + f := newFp2() + a, _ := new(fe2).rand(rand.Reader) + b0, b1, b2, b3 := new(fe2), new(fe2), new(fe2), new(fe2) + f.exp(b0, a, modulus.big()) + fp2Conjugate(b1, a) + b2.set(a) + f.frobeniusMap1(b2) + b3.set(a) + f.frobeniusMap(b3, 1) + if !b0.equal(b3) { + t.Fatal("frobenius map failed") + } + if !b1.equal(b3) { + t.Fatal("frobenius map failed") + } + if !b2.equal(b3) { + t.Fatal("frobenius map failed") + } +} + +func TestFrobeniusMapping6(t *testing.T) { + { + f := newFp2() + z := nonResidue2 + for i := 0; i < 6; i++ { + p, r, e := modulus.big(), new(fe2), big.NewInt(0) + // p ^ i + p.Exp(p, big.NewInt(int64(i)), nil) + // (p ^ i - 1) / 3 + e.Sub(p, big.NewInt(1)).Div(e, big.NewInt(3)) + // r = z ^ (p ^ i - 1) / 3 + f.exp(r, z, e) + if !r.equal(&frobeniusCoeffs61[i]) { + t.Fatalf("bad frobenius fp6 1q coefficient") + } + } + for i := 0; i < 6; i++ { + p, r, e := modulus.big(), new(fe2), big.NewInt(0) + // p ^ i + p.Exp(p, big.NewInt(int64(i)), nil).Mul(p, big.NewInt(2)) + // (2 * p ^ i - 2) / 3 + e.Sub(p, big.NewInt(2)).Div(e, big.NewInt(3)) + // r = z ^ (2 * p ^ i - 2) / 3 + f.exp(r, z, e) + if !r.equal(&frobeniusCoeffs62[i]) { + t.Fatalf("bad frobenius fp6 2q coefficient") + } + } + } + f := newFp6(nil) + r0, r1 := f.new(), f.new() + e, _ := new(fe6).rand(rand.Reader) + r0.set(e) + r1.set(e) + f.frobeniusMap(r1, 1) + f.frobeniusMap1(r0) + if !r0.equal(r1) { + t.Fatalf("frobenius mapping by 1 failed") + } + r0.set(e) + r1.set(e) + f.frobeniusMap(r1, 2) + f.frobeniusMap2(r0) + if !r0.equal(r1) { + t.Fatalf("frobenius mapping by 2 failed") + } + r0.set(e) + r1.set(e) + f.frobeniusMap(r1, 3) + f.frobeniusMap3(r0) + if !r0.equal(r1) { + t.Fatalf("frobenius mapping by 3 failed") + } +} + +func TestFrobeniusMapping12(t *testing.T) { + { + f := newFp2() + z := nonResidue2 + for i := 0; i < 12; i++ { + p, r, e := modulus.big(), new(fe2), big.NewInt(0) + // p ^ i + p.Exp(p, big.NewInt(int64(i)), nil) + // (p ^ i - 1) / 6 + e.Sub(p, big.NewInt(1)).Div(e, big.NewInt(6)) + // r = z ^ (p ^ i - 1) / 6 + f.exp(r, z, e) + if !r.equal(&frobeniusCoeffs12[i]) { + t.Fatalf("bad frobenius fp12 coefficient") + } + } + } + f := newFp12(nil) + r0, r1 := f.new(), f.new() + e, _ := new(fe12).rand(rand.Reader) + p := modulus.big() + f.exp(r0, e, p) + r1.set(e) + f.frobeniusMap1(r1) + if !r0.equal(r1) { + t.Fatalf("frobenius mapping by 1 failed") + } + p.Mul(p, modulus.big()) + f.exp(r0, e, p) + r1.set(e) + f.frobeniusMap2(r1) + if !r0.equal(r1) { + t.Fatalf("frobenius mapping by 2 failed") + } + p.Mul(p, modulus.big()) + f.exp(r0, e, p) + r1.set(e) + f.frobeniusMap3(r1) + if !r0.equal(r1) { + t.Fatalf("frobenius mapping by 2 failed") + } +} + +func TestFp12MultiplicationCross(t *testing.T) { + f := newFp12(nil) + a, b, c0, c1, c2 := new(fe12), new(fe12), new(fe12), new(fe12), new(fe12) + for i := 0; i < fuz; i++ { + _, _ = a.rand(rand.Reader) + _, _ = b.rand(rand.Reader) + f.mul(c0, a, b) + c1.set(a) + f.mulAssign(c1, b) + f._mul(c2, a, b) + + if !c0.equal(c1) { + t.Fatal("multiplication failed") + } + if !c0.equal(c2) { + t.Fatal("multiplication failed") + } + } +} + +func TestFp12SparseMultiplicationCross(t *testing.T) { + f := newFp12(nil) + a, c0, c1 := new(fe12), new(fe12), new(fe12) + + for i := 0; i < fuz; i++ { + _, _ = a.rand(rand.Reader) + b0, _ := new(fe2).rand(rand.Reader) + b1, _ := new(fe2).rand(rand.Reader) + b4, _ := new(fe2).rand(rand.Reader) + b := new(fe12) + b[0][0].set(b0) + b[0][1].set(b1) + b[1][1].set(b4) + + c0.set(a) + f.mul014(c0, b0, b1, b4) + f._mul(c1, a, b) + + if !c0.equal(c1) { + t.Fatal("sparse multiplication 014 failed") + } + } +} + +func TestFp4MultiplicationCross(t *testing.T) { + f := newFp12(nil) + a0, a1, b0, b1 := new(fe2), new(fe2), new(fe2), new(fe2) + c0, c1 := new(fe2), new(fe2) + + for i := 0; i < fuz; i++ { + _, _ = a0.rand(rand.Reader) + _, _ = a1.rand(rand.Reader) + _, _ = b0.rand(rand.Reader) + _, _ = b1.rand(rand.Reader) + c0.set(a0) + c1.set(a1) + + f._fp4Square(a0, a1, b0, b1) + f.fp4Square(c0, c1, b0, b1) + + if !a0.equal(c0) { + t.Fatal("fp4 multiplication failed") + } + if !a1.equal(c1) { + t.Fatal("fp4 multiplication failed") + } + } +} + +func BenchmarkFpMul(t *testing.B) { a, _ := new(fe).rand(rand.Reader) b, _ := new(fe).rand(rand.Reader) - c, _ := new(fe).rand(rand.Reader) + c := new(fe) t.ResetTimer() for i := 0; i < t.N; i++ { mul(c, a, b) } } -func BenchmarkInverse(t *testing.B) { - a, _ := new(fe).rand(rand.Reader) - b, _ := new(fe).rand(rand.Reader) - t.ResetTimer() - for i := 0; i < t.N; i++ { - inverse(a, b) +func (fe *wfe) bytes() []byte { + out := make([]byte, fpByteSize*2) + var a int + for i := 0; i < 2*fpNumberOfLimbs; i++ { + a = fpByteSize*2 - i*8 + out[a-1] = byte(fe[i]) + out[a-2] = byte(fe[i] >> 8) + out[a-3] = byte(fe[i] >> 16) + out[a-4] = byte(fe[i] >> 24) + out[a-5] = byte(fe[i] >> 32) + out[a-6] = byte(fe[i] >> 40) + out[a-7] = byte(fe[i] >> 48) + out[a-8] = byte(fe[i] >> 56) } -} - -func padBytes(in []byte, size int) []byte { - out := make([]byte, size) - if len(in) > size { - panic("bad input for padding") - } - copy(out[size-len(in):], in) return out } + +func (fe *wfe) equal(fe2 *wfe) bool { + return fe2[0] == fe[0] && fe2[1] == fe[1] && fe2[2] == fe[2] && fe2[3] == fe[3] && fe2[4] == fe[4] && fe2[5] == fe[5] && fe2[6] == fe[6] && fe2[7] == fe[7] && fe2[8] == fe[8] && fe2[9] == fe[9] && fe2[10] == fe[10] && fe2[11] == fe[11] +} + +func (fe *wfe2) equal(fe2 *wfe2) bool { + return fe[0].equal(&fe2[0]) && fe[1].equal(&fe2[1]) +} + +func _fp2MulByNonResidue(c, a *fe2) { + t0 := &fe{} + add(t0, &a[0], &a[1]) + sub(&c[0], &a[0], &a[1]) + c[1].set(t0) +} + +func _wfp2Add(c, a, b *wfe2) { + wadd(&c[0], &a[0], &b[0]) + wadd(&c[1], &a[1], &b[1]) +} + +func _wfp2Ladd(c, a, b *wfe2) { + lwadd(&c[0], &a[0], &b[0]) + lwadd(&c[1], &a[1], &b[1]) +} + +func _wfp2AddMixed(c, a, b *wfe2) { + wadd(&c[0], &a[0], &b[0]) + lwadd(&c[1], &a[1], &b[1]) +} + +func _wfp2Sub(c, a, b *wfe2) { + wsub(&c[0], &a[0], &b[0]) + wsub(&c[1], &a[1], &b[1]) +} + +func _wfp2SubMixed(c, a, b *wfe2) { + wsub(&c[0], &a[0], &b[0]) + lwsub(&c[1], &a[1], &b[1]) +} + +func _wfp2Double(c, a *wfe2) { + wdouble(&c[0], &a[0]) + wdouble(&c[1], &a[1]) +} + +func _wfp2MulByNonResidue(c, a *wfe2) { + wt0 := &wfe{} + wadd(wt0, &a[0], &a[1]) + wsub(&c[0], &a[0], &a[1]) + c[1].set(wt0) +} + +func _wfp2Mul(c *wfe2, a, b *fe2) { + wt0, wt1 := new(wfe), new(wfe) + t0, t1 := new(fe), new(fe) + wmul(wt0, &a[0], &b[0]) // a0b0 + wmul(wt1, &a[1], &b[1]) // a1b1 + wsub(&c[0], wt0, wt1) // c0 = a0b0 - a1b1 + lwaddAssign(wt0, wt1) // a0b0 + a1b1 + ladd(t0, &a[0], &a[1]) // a0 + a1 + ladd(t1, &b[0], &b[1]) // b0 + b1 + wmul(wt1, t0, t1) // (a0 + a1)(b0 + b1) + lwsub(&c[1], wt1, wt0) // c1 = (a0 + a1)(b0 + b1) - (a0b0 + a1b1) +} + +func _wfp2Square(c *wfe2, a *fe2) { + t0, t1, t2 := new(fe), new(fe), new(fe) + ladd(t0, &a[0], &a[1]) // (a0 + a1) + sub(t1, &a[0], &a[1]) // (a0 - a1) + ldouble(t2, &a[0]) // 2a0 + wmul(&c[0], t1, t0) // c0 = (a0 + a1)(a0 - a1) + wmul(&c[1], t2, &a[1]) // c1 = 2a0a1 +} + +func (e *fp6) _mul(c, a, b *fe6) { + t0, t1, t2, t3, t4, t5 := new(fe2), new(fe2), new(fe2), new(fe2), new(fe2), new(fe2) + e.fp2.mul(t0, &a[0], &b[0]) // v0 = a0b0 + e.fp2.mul(t1, &a[1], &b[1]) // v1 = a1b1 + e.fp2.mul(t2, &a[2], &b[2]) // v2 = a2b2 + fp2Add(t3, &a[1], &a[2]) // a1 + a2 + fp2Add(t4, &b[1], &b[2]) // b1 + b2 + e.fp2.mulAssign(t3, t4) // (a1 + a2)(b1 + b2) + fp2Add(t4, t1, t2) // v1 + v2 + fp2SubAssign(t3, t4) // (a1 + a2)(b1 + b2) - v1 - v2 + mulByNonResidueAssign(t3) // ((a1 + a2)(b1 + b2) - v1 - v2)β + fp2AddAssign(t3, t0) // c0 = ((a1 + a2)(b1 + b2) - v1 - v2)β + v0 + fp2Add(t5, &a[0], &a[1]) // a0 + a1 + fp2Add(t4, &b[0], &b[1]) // b0 + b1 + e.fp2.mulAssign(t5, t4) // (a0 + a1)(b0 + b1) + fp2Add(t4, t0, t1) // v0 + v1 + fp2SubAssign(t5, t4) // (a0 + a1)(b0 + b1) - v0 - v1 + mulByNonResidue(t4, t2) // βv2 + fp2Add(&c[1], t5, t4) // c1 = (a0 + a1)(b0 + b1) - v0 - v1 + βv2 + fp2Add(t5, &a[0], &a[2]) // a0 + a2 + fp2Add(t4, &b[0], &b[2]) // b0 + b2 + e.fp2.mulAssign(t5, t4) // (a0 + a2)(b0 + b2) + fp2Add(t4, t0, t2) // v0 + v2 + fp2SubAssign(t5, t4) // (a0 + a2)(b0 + b2) - v0 - v2 + fp2Add(&c[2], t1, t5) // c2 = (a0 + a2)(b0 + b2) - v0 - v2 + v1 + c[0].set(t3) +} + +func (e *fp6) _mul01(c, a *fe6, b0, b1 *fe2) { + t0, t1, t2, t3, t4 := new(fe2), new(fe2), new(fe2), new(fe2), new(fe2) + e.fp2.mul(t0, &a[0], b0) // v0 = b0a0 + e.fp2.mul(t1, &a[1], b1) // v1 = a1b1 + fp2Add(t2, &a[1], &a[2]) // a1 + a2 + e.fp2.mulAssign(t2, b1) // b1(a1 + a2) + fp2SubAssign(t2, t1) // b1(a1 + a2) - v1 + mulByNonResidueAssign(t2) // (b1(a1 + a2) - v1)β + fp2Add(t3, &a[0], &a[2]) // a0 + a2 + e.fp2.mulAssign(t3, b0) // b0(a0 + a2) + fp2SubAssign(t3, t0) // b0(a0 + a2) - v0 + fp2Add(&c[2], t3, t1) // b0(a0 + a2) - v0 + v1 + fp2Add(t4, b0, b1) // (b0 + b1) + fp2Add(t3, &a[0], &a[1]) // (a0 + a1) + e.fp2.mulAssign(t4, t3) // (a0 + a1)(b0 + b1) + fp2SubAssign(t4, t0) // (a0 + a1)(b0 + b1) - v0 + fp2Sub(&c[1], t4, t1) // (a0 + a1)(b0 + b1) - v0 - v1 + fp2Add(&c[0], t2, t0) // (b1(a1 + a2) - v1)β + v0 +} + +func (e *fp6) _mul1(c, a *fe6, b1 *fe2) { + t := new(fe2) + e.fp2.mul(t, &a[2], b1) + e.fp2.mul(&c[2], &a[1], b1) + e.fp2.mul(&c[1], &a[0], b1) + mulByNonResidue(&c[0], t) +} + +func (e *fp6) _square(c, a *fe6) { + t0, t1, t2, t3, t4, t5 := new(fe2), new(fe2), new(fe2), new(fe2), new(fe2), new(fe2) + e.fp2.square(t0, &a[0]) + e.fp2.mul(t1, &a[0], &a[1]) + fp2DoubleAssign(t1) + fp2Sub(t2, &a[0], &a[1]) + fp2AddAssign(t2, &a[2]) + e.fp2.squareAssign(t2) + e.fp2.mul(t3, &a[1], &a[2]) + fp2DoubleAssign(t3) + e.fp2.square(t4, &a[2]) + mulByNonResidue(t5, t3) + fp2Add(&c[0], t0, t5) + mulByNonResidue(t5, t4) + fp2Add(&c[1], t1, t5) + fp2AddAssign(t1, t2) + fp2AddAssign(t1, t3) + fp2AddAssign(t0, t4) + fp2Sub(&c[2], t1, t0) +} + +func (e *fp12) _mul(c, a, b *fe12) { + t0, t1, t2, t3 := new(fe6), new(fe6), new(fe6), new(fe6) + e.fp6.mul(t1, &a[0], &b[0]) // v0 = a0b0 + e.fp6.mul(t2, &a[1], &b[1]) // v1 = a1b1 + fp6Add(t0, &a[0], &a[1]) // a0 + a1 + fp6Add(t3, &b[0], &b[1]) // b0 + b1 + e.fp6.mulAssign(t0, t3) // (a0 + a1)(b0 + b1) + fp6SubAssign(t0, t1) // (a0 + a1)(b0 + b1) - v0 + fp6Sub(&c[1], t0, t2) // c1 = (a0 + a1)(b0 + b1) - v0 - v1 + e.fp6.mulByNonResidue(t2, t2) // βv1 + fp6Add(&c[0], t1, t2) // c0 = v0 + βv1 +} + +func (e *fp12) _fp4Square(c0, c1, a0, a1 *fe2) { + t, fp2 := e.t2, e.fp2() + + fp2.square(t[0], a0) // a0^2 + fp2.square(t[1], a1) // a1^2 + mulByNonResidue(t[2], t[1]) // βa1^2 + fp2Add(c0, t[2], t[0]) // c0 = βa1^2 + a0^2 + fp2Add(t[2], a0, a1) // a0 + a1 + fp2.squareAssign(t[2]) // (a0 + a1)^2 + fp2SubAssign(t[2], t[0]) // (a0 + a1)^2 - a0^2 + fp2Sub(c1, t[2], t[1]) // (a0 + a1)^2 - a0^2 - a1^2 +} diff --git a/crypto/bls12381/fr.go b/crypto/bls12381/fr.go new file mode 100644 index 0000000000..92d825f7f9 --- /dev/null +++ b/crypto/bls12381/fr.go @@ -0,0 +1,455 @@ +package bls12381 + +import ( + "crypto/rand" + "io" + "math/big" + "math/bits" +) + +const frByteSize = 32 +const frBitSize = 255 +const frNumberOfLimbs = 4 +const fourWordBitSize = 256 + +type Fr [4]uint64 +type wideFr [8]uint64 + +func NewFr() *Fr { + return &Fr{} +} + +func (e *Fr) Rand(r io.Reader) (*Fr, error) { + bi, err := rand.Int(r, qBig) + if err != nil { + return nil, err + } + _ = e.fromBig(bi) + return e, nil +} + +func (e *Fr) Set(e2 *Fr) *Fr { + e[0] = e2[0] + e[1] = e2[1] + e[2] = e2[2] + e[3] = e2[3] + return e +} + +func (e *Fr) Zero() *Fr { + e[0] = 0 + e[1] = 0 + e[2] = 0 + e[3] = 0 + return e +} + +func (e *Fr) One() *Fr { + e.Set(&Fr{1}) + return e +} + +func (e *Fr) RedOne() *Fr { + e.Set(qr1) + return e +} + +func (e *Fr) FromBytes(in []byte) *Fr { + e.fromBytes(in) + return e +} + +func (e *Fr) RedFromBytes(in []byte) *Fr { + e.fromBytes(in) + e.toMont() + return e +} + +func (e *Fr) fromBytes(in []byte) *Fr { + u := new(big.Int).SetBytes(in) + _ = e.fromBig(u) + return e +} + +func (e *Fr) fromBig(in *big.Int) *Fr { + e.Zero() + _in := new(big.Int).Set(in) + zero := new(big.Int) + c0 := _in.Cmp(zero) + c1 := _in.Cmp(qBig) + if c0 == -1 || c1 == 1 { + _in.Mod(_in, qBig) + } + + words := _in.Bits() // a little-endian Word slice + if bits.UintSize == 64 { // in the 64-bit architecture + for i := 0; i < len(words); i++ { + e[i] = uint64(words[i]) + } + } else { // in the 32-bit architecture + for i := 0; i < len(e); i++ { + j := i * 2 + if j+1 < len(words) { + e[i] = uint64(words[j+1])<<32 | uint64(words[j]) + } else if j < len(words) { + e[i] = uint64(words[j]) + } else { + e[i] = uint64(0) + } + } + } + + return e +} + +func (e *Fr) setUint64(n uint64) *Fr { + e.Zero() + e[0] = n + return e +} + +func (e *Fr) ToBytes() []byte { + return NewFr().Set(e).bytes() +} + +func (e *Fr) RedToBytes() []byte { + out := NewFr().Set(e) + out.fromMont() + return out.bytes() +} + +func (e *Fr) ToBig() *big.Int { + return new(big.Int).SetBytes(e.ToBytes()) +} + +func (e *Fr) RedToBig() *big.Int { + return new(big.Int).SetBytes(e.RedToBytes()) +} + +func (e *Fr) bytes() []byte { + out := make([]byte, frByteSize) + var a int + for i := 0; i < frNumberOfLimbs; i++ { + a = frByteSize - i*8 + out[a-1] = byte(e[i]) + out[a-2] = byte(e[i] >> 8) + out[a-3] = byte(e[i] >> 16) + out[a-4] = byte(e[i] >> 24) + out[a-5] = byte(e[i] >> 32) + out[a-6] = byte(e[i] >> 40) + out[a-7] = byte(e[i] >> 48) + out[a-8] = byte(e[i] >> 56) + } + return out +} + +func (e *Fr) IsZero() bool { + return (e[3] | e[2] | e[1] | e[0]) == 0 +} + +func (e *Fr) IsOne() bool { + return e.Equal(&Fr{1}) +} + +func (e *Fr) IsRedOne() bool { + return e.Equal(qr1) +} + +func (e *Fr) Equal(e2 *Fr) bool { + return e2[0] == e[0] && e2[1] == e[1] && e2[2] == e[2] && e2[3] == e[3] +} + +func (e *Fr) Cmp(e1 *Fr) int { + for i := frNumberOfLimbs - 1; i >= 0; i-- { + if e[i] > e1[i] { + return 1 + } else if e[i] < e1[i] { + return -1 + } + } + return 0 +} + +func (e *Fr) sliceUint64(from int) uint64 { + if from < 64 { + return e[0]>>from | e[1]<<(64-from) + } else if from < 128 { + return e[1]>>(from-64) | e[2]<<(128-from) + } else if from < 192 { + return e[2]>>(from-128) | e[3]<<(192-from) + } + return e[3] >> (from - 192) +} + +func (e *Fr) div2() { + e[0] = e[0]>>1 | e[1]<<63 + e[1] = e[1]>>1 | e[2]<<63 + e[2] = e[2]>>1 | e[3]<<63 + e[3] = e[3] >> 1 +} + +func (e *Fr) mul2() uint64 { + c := e[3] >> 63 + e[3] = e[3]<<1 | e[2]>>63 + e[2] = e[2]<<1 | e[1]>>63 + e[1] = e[1]<<1 | e[0]>>63 + e[0] = e[0] << 1 + return c +} + +func (e *Fr) isEven() bool { + var mask uint64 = 1 + return e[0]&mask == 0 +} + +func (e *Fr) Bit(at int) bool { + if at < 64 { + return (e[0]>>at)&1 == 1 + } else if at < 128 { + return (e[1]>>(at-64))&1 == 1 + } else if at < 192 { + return (e[2]>>(at-128))&1 == 1 + } else if at < 256 { + return (e[3]>>(at-192))&1 == 1 + } + return false +} + +func (e *Fr) toMont() { + e.RedMul(e, qr2) +} + +func (e *Fr) fromMont() { + e.RedMul(e, &Fr{1}) +} + +func (e *Fr) FromRed() { + e.fromMont() +} + +func (e *Fr) ToRed() { + e.toMont() +} + +func (e *Fr) Add(a, b *Fr) { + addFR(e, a, b) +} + +func (e *Fr) Double(a *Fr) { + doubleFR(e, a) +} + +func (e *Fr) Sub(a, b *Fr) { + subFR(e, a, b) +} + +func (e *Fr) Neg(a *Fr) { + negFR(e, a) +} + +func (e *Fr) Mul(a, b *Fr) { + e.RedMul(a, b) + e.toMont() +} + +func (e *Fr) RedMul(a, b *Fr) { + mulFR(e, a, b) +} + +func (e *Fr) Square(a *Fr) { + e.RedSquare(a) + e.toMont() +} + +func (e *Fr) RedSquare(a *Fr) { + squareFR(e, a) +} + +func (e *Fr) RedExp(a *Fr, ee *big.Int) { + z := new(Fr).RedOne() + for i := ee.BitLen(); i >= 0; i-- { + z.RedSquare(z) + if ee.Bit(i) == 1 { + z.RedMul(z, a) + } + } + e.Set(z) +} + +func (e *Fr) Exp(a *Fr, ee *big.Int) { + e.Set(a).toMont() + e.RedExp(e, ee) + e.fromMont() + +} + +func RedInverseBatchFr(in []Fr) { + inverseBatchFr(in, func(a, b *Fr) { a.RedInverse(b) }) +} + +func InverseBatchFr(in []Fr) { + inverseBatchFr(in, func(a, b *Fr) { a.Inverse(b) }) +} + +func inverseBatchFr(in []Fr, invFn func(out *Fr, in *Fr)) { + n, N, setFirst := 0, len(in), false + + for i := 0; i < len(in); i++ { + if !in[i].IsZero() { + n++ + } + } + if n == 0 { + return + } + + tA := make([]Fr, n) + tB := make([]Fr, n) + + for i, j := 0, 0; i < N; i++ { + if !in[i].IsZero() { + if !setFirst { + setFirst = true + tA[j].Set(&in[i]) + } else { + tA[j].Mul(&in[i], &tA[j-1]) + } + j = j + 1 + } + } + + invFn(&tB[n-1], &tA[n-1]) + for i, j := N-1, n-1; j != 0; i-- { + if !in[i].IsZero() { + tB[j-1].Mul(&tB[j], &in[i]) + j = j - 1 + } + } + + for i, j := 0, 0; i < N; i++ { + if !in[i].IsZero() { + if setFirst { + setFirst = false + in[i].Set(&tB[j]) + } else { + in[i].Mul(&tA[j-1], &tB[j]) + } + j = j + 1 + } + } +} + +func (e *Fr) Inverse(a *Fr) { + e.Set(a).toMont() + e.RedInverse(e) + e.fromMont() +} + +func (e *Fr) RedInverse(ei *Fr) { + if ei.IsZero() { + e.Zero() + return + } + u := new(Fr).Set(&q) + v := new(Fr).Set(ei) + s := &Fr{1} + r := &Fr{0} + var k int + var z uint64 + var found = false + // Phase 1 + for i := 0; i < fourWordBitSize*2; i++ { + if v.IsZero() { + found = true + break + } + if u.isEven() { + u.div2() + s.mul2() + } else if v.isEven() { + v.div2() + z += r.mul2() + } else if u.Cmp(v) == 1 { + lsubAssignFR(u, v) + u.div2() + laddAssignFR(r, s) + s.mul2() + } else { + lsubAssignFR(v, u) + v.div2() + laddAssignFR(s, r) + z += r.mul2() + } + k += 1 + } + + if !found { + e.Zero() + return + } + + if k < frBitSize || k > frBitSize+fourWordBitSize { + e.Zero() + return + } + + if r.Cmp(&q) != -1 || z > 0 { + lsubAssignFR(r, &q) + } + u.Set(&q) + lsubAssignFR(u, r) + + // Phase 2 + for i := k; i < 2*fourWordBitSize; i++ { + doubleFR(u, u) + } + e.Set(u) +} + +func (ew *wideFr) mul(a, b *Fr) { + wmulFR(ew, a, b) +} + +func (ew *wideFr) add(a *wideFr) { + waddFR(ew, a) +} + +func (ew *wideFr) round() *Fr { + ew.add(halfR) + return ew.high() +} + +func (ew *wideFr) high() *Fr { + e := new(Fr) + e[0] = ew[4] + e[1] = ew[5] + e[2] = ew[6] + e[3] = ew[7] + return e +} + +func (ew *wideFr) low() *Fr { + e := new(Fr) + e[0] = ew[0] + e[1] = ew[1] + e[2] = ew[2] + e[3] = ew[3] + return e +} + +func (e *wideFr) bytes() []byte { + out := make([]byte, frByteSize*2) + var a int + for i := 0; i < frNumberOfLimbs*2; i++ { + a = frByteSize*2 - i*8 + out[a-1] = byte(e[i]) + out[a-2] = byte(e[i] >> 8) + out[a-3] = byte(e[i] >> 16) + out[a-4] = byte(e[i] >> 24) + out[a-5] = byte(e[i] >> 32) + out[a-6] = byte(e[i] >> 40) + out[a-7] = byte(e[i] >> 48) + out[a-8] = byte(e[i] >> 56) + } + return out +} diff --git a/crypto/bls12381/fr_arithmetic_x86.s b/crypto/bls12381/fr_arithmetic_x86.s new file mode 100644 index 0000000000..0c421a0213 --- /dev/null +++ b/crypto/bls12381/fr_arithmetic_x86.s @@ -0,0 +1,1411 @@ +// +build amd64,!generic + +#include "textflag.h" +#include "funcdata.h" + + +// func addFR(c *[4]uint64, a *[4]uint64, b *[4]uint64) +TEXT ·addFR(SB), NOSPLIT, $0-24 + // | + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + + // | + MOVQ (DI), CX + MOVQ 8(DI), DX + MOVQ 16(DI), R8 + MOVQ 24(DI), R9 + ADDQ (SI), CX + ADCQ 8(SI), DX + ADCQ 16(SI), R8 + ADCQ 24(SI), R9 + + // | + MOVQ CX, R10 + MOVQ DX, R11 + MOVQ R8, R12 + MOVQ R9, R13 + SUBQ ·q+0(SB), R10 + SBBQ ·q+8(SB), R11 + SBBQ ·q+16(SB), R12 + SBBQ ·q+24(SB), R13 + + // | + MOVQ c+0(FP), DI + CMOVQCC R10, CX + CMOVQCC R11, DX + CMOVQCC R12, R8 + CMOVQCC R13, R9 + MOVQ CX, (DI) + MOVQ DX, 8(DI) + MOVQ R8, 16(DI) + MOVQ R9, 24(DI) + RET +/* end */ + + +// func laddAssignFR(a *[4]uint64, b *[4]uint64) +TEXT ·laddAssignFR(SB), NOSPLIT, $0-16 + // | + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + // | + MOVQ (DI), CX + MOVQ 8(DI), DX + MOVQ 16(DI), R8 + MOVQ 24(DI), R9 + ADDQ (SI), CX + ADCQ 8(SI), DX + ADCQ 16(SI), R8 + ADCQ 24(SI), R9 + MOVQ CX, (DI) + MOVQ DX, 8(DI) + MOVQ R8, 16(DI) + MOVQ R9, 24(DI) + RET +/* end */ + + +// func doubleFR(c *[4]uint64, a *[4]uint64) +TEXT ·doubleFR(SB), NOSPLIT, $0-16 + // | + MOVQ a+8(FP), DI + + MOVQ (DI), CX + MOVQ 8(DI), DX + MOVQ 16(DI), SI + MOVQ 24(DI), R8 + ADDQ CX, CX + ADCQ DX, DX + ADCQ SI, SI + ADCQ R8, R8 + + // | + MOVQ CX, R9 + MOVQ DX, R10 + MOVQ SI, R11 + MOVQ R8, R12 + SUBQ ·q+0(SB), R9 + SBBQ ·q+8(SB), R10 + SBBQ ·q+16(SB), R11 + SBBQ ·q+24(SB), R12 + + // | + MOVQ c+0(FP), DI + CMOVQCC R9, CX + CMOVQCC R10, DX + CMOVQCC R11, SI + CMOVQCC R12, R8 + MOVQ CX, (DI) + MOVQ DX, 8(DI) + MOVQ SI, 16(DI) + MOVQ R8, 24(DI) + RET +/* end */ + + +// func subFR(c *[4]uint64, a *[4]uint64, b *[4]uint64) +TEXT ·subFR(SB), NOSPLIT, $0-24 + // | + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + XORQ AX, AX + + MOVQ (DI), CX + MOVQ 8(DI), DX + MOVQ 16(DI), R8 + MOVQ 24(DI), R9 + SUBQ (SI), CX + SBBQ 8(SI), DX + SBBQ 16(SI), R8 + SBBQ 24(SI), R9 + + // | + MOVQ ·q+0(SB), SI + MOVQ ·q+8(SB), R10 + MOVQ ·q+16(SB), R11 + MOVQ ·q+24(SB), R12 + CMOVQCC AX, SI + CMOVQCC AX, R10 + CMOVQCC AX, R11 + CMOVQCC AX, R12 + + // | + ADDQ SI, CX + ADCQ R10, DX + ADCQ R11, R8 + ADCQ R12, R9 + + MOVQ c+0(FP), DI + MOVQ CX, (DI) + MOVQ DX, 8(DI) + MOVQ R8, 16(DI) + MOVQ R9, 24(DI) + RET +/* end */ + + +// func lsubAssignFR(a *[4]uint64, b *[4]uint64) +TEXT ·lsubAssignFR(SB), NOSPLIT, $0-16 + // | + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + MOVQ (DI), CX + MOVQ 8(DI), DX + MOVQ 16(DI), R8 + MOVQ 24(DI), R9 + SUBQ (SI), CX + SBBQ 8(SI), DX + SBBQ 16(SI), R8 + SBBQ 24(SI), R9 + MOVQ CX, (DI) + MOVQ DX, 8(DI) + MOVQ R8, 16(DI) + MOVQ R9, 24(DI) + RET +/* end */ + + +// func _negFR(c *[4]uint64, a *[4]uint64) +TEXT ·_negFR(SB), NOSPLIT, $0-16 + // | + MOVQ a+8(FP), DI + + // | + MOVQ ·q+0(SB), CX + SUBQ (DI), CX + MOVQ ·q+8(SB), DX + SBBQ 8(DI), DX + MOVQ ·q+16(SB), SI + SBBQ 16(DI), SI + MOVQ ·q+24(SB), R8 + SBBQ 24(DI), R8 + + // | + MOVQ c+0(FP), DI + MOVQ CX, (DI) + MOVQ DX, 8(DI) + MOVQ SI, 16(DI) + MOVQ R8, 24(DI) + RET +/* end */ + + +// func mulFR(c *[4]uint64, a *[4]uint64, b *[4]uint64) +TEXT ·mulNoADXFR(SB), NOSPLIT, $0-24 + // | + +/* inputs */ + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ $0x00, R10 + MOVQ $0x00, R11 + MOVQ $0x00, R12 + MOVQ $0x00, R13 + MOVQ $0x00, R14 + + // | + +/* i = 0 */ + + // | a0 @ CX + MOVQ (DI), CX + + // | a0 * b0 + MOVQ (SI), AX + MULQ CX + MOVQ AX, R8 + MOVQ DX, R9 + + // | a0 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + + // | a0 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + + // | a0 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + + // | + +/* i = 1 */ + + // | a1 @ CX + MOVQ 8(DI), CX + MOVQ $0x00, BX + + // | a1 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + ADCQ $0x00, R11 + ADCQ $0x00, BX + + // | a1 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ BX, R12 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a1 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + + // | a1 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + + // | + +/* i = 2 */ + + // | a2 @ CX + MOVQ 16(DI), CX + MOVQ $0x00, BX + + // | a2 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ $0x00, R12 + ADCQ $0x00, BX + + // | a2 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a2 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + + // | a2 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + + // | + +/* i = 3 */ + + // | a3 @ CX + MOVQ 24(DI), CX + MOVQ $0x00, BX + + // | a3 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ $0x00, R13 + ADCQ $0x00, BX + + // | a3 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a3 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ $0x00, BX + + // | a3 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, BX + + // | + +/* */ + + // | + // | W + // | 0 R8 | 1 R9 | 2 R10 | 3 R11 + // | 4 R12 | 5 R13 | 6 R14 | 7 BX + + + // | + +/* montgomery reduction */ + + // | + +/* i = 0 */ + + // | + // | W + // | 0 R8 | 1 R9 | 2 R10 | 3 R11 + // | 4 R12 | 5 R13 | 6 R14 | 7 BX + + + // | | u0 = w0 * inp + MOVQ R8, AX + MULQ ·qinp+0(SB) + MOVQ AX, DI + MOVQ $0x00, CX + + // | + +/* */ + + // | j0 + + // | w0 @ R8 + MOVQ ·q+0(SB), AX + MULQ DI + ADDQ AX, R8 + ADCQ DX, CX + + // | j1 + + // | w1 @ R9 + MOVQ ·q+8(SB), AX + MULQ DI + ADDQ AX, R9 + ADCQ $0x00, DX + ADDQ CX, R9 + MOVQ $0x00, CX + ADCQ DX, CX + + // | j2 + + // | w2 @ R10 + MOVQ ·q+16(SB), AX + MULQ DI + ADDQ AX, R10 + ADCQ $0x00, DX + ADDQ CX, R10 + MOVQ $0x00, CX + ADCQ DX, CX + + // | j3 + + // | w3 @ R11 + MOVQ ·q+24(SB), AX + MULQ DI + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ CX, R11 + + // | w4 @ R12 + ADCQ DX, R12 + ADCQ $0x00, R8 + + // | + +/* i = 1 */ + + // | + // | W + // | 0 - | 1 R9 | 2 R10 | 3 R11 + // | 4 R12 | 5 R13 | 6 R14 | 7 BX + + + // | | u1 = w1 * inp + MOVQ R9, AX + MULQ ·qinp+0(SB) + MOVQ AX, DI + MOVQ $0x00, CX + + // | + +/* */ + + // | j0 + + // | w1 @ R9 + MOVQ ·q+0(SB), AX + MULQ DI + ADDQ AX, R9 + ADCQ DX, CX + + // | j1 + + // | w2 @ R10 + MOVQ ·q+8(SB), AX + MULQ DI + ADDQ AX, R10 + ADCQ $0x00, DX + ADDQ CX, R10 + MOVQ $0x00, CX + ADCQ DX, CX + + // | j2 + + // | w3 @ R11 + MOVQ ·q+16(SB), AX + MULQ DI + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ CX, R11 + MOVQ $0x00, CX + ADCQ DX, CX + + // | j3 + + // | w4 @ R12 + MOVQ ·q+24(SB), AX + MULQ DI + ADDQ AX, R12 + ADCQ DX, R8 + ADDQ CX, R12 + + // | w5 @ R13 + ADCQ R8, R13 + MOVQ $0x00, R8 + ADCQ $0x00, R8 + + // | + +/* i = 2 */ + + // | + // | W + // | 0 - | 1 - | 2 R10 | 3 R11 + // | 4 R12 | 5 R13 | 6 R14 | 7 BX + + + // | | u2 = w2 * inp + MOVQ R10, AX + MULQ ·qinp+0(SB) + MOVQ AX, DI + MOVQ $0x00, CX + + // | + +/* */ + + // | j0 + + // | w2 @ R10 + MOVQ ·q+0(SB), AX + MULQ DI + ADDQ AX, R10 + ADCQ DX, CX + + // | j1 + + // | w3 @ R11 + MOVQ ·q+8(SB), AX + MULQ DI + ADDQ AX, R11 + ADCQ $0x00, DX + ADDQ CX, R11 + MOVQ $0x00, CX + ADCQ DX, CX + + // | j2 + + // | w4 @ R12 + MOVQ ·q+16(SB), AX + MULQ DI + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ CX, R12 + MOVQ $0x00, CX + ADCQ DX, CX + + // | j3 + + // | w5 @ R13 + MOVQ ·q+24(SB), AX + MULQ DI + ADDQ AX, R13 + ADCQ DX, R8 + ADDQ CX, R13 + + // | w6 @ R14 + ADCQ R8, R14 + MOVQ $0x00, R8 + ADCQ $0x00, R8 + + // | + +/* i = 3 */ + + // | + // | W + // | 0 - | 1 - | 2 - | 3 R11 + // | 4 R12 | 5 R13 | 6 R14 | 7 BX + + + // | | u3 = w3 * inp + MOVQ R11, AX + MULQ ·qinp+0(SB) + MOVQ AX, DI + MOVQ $0x00, CX + + // | + +/* */ + + // | j0 + + // | w3 @ R11 + MOVQ ·q+0(SB), AX + MULQ DI + ADDQ AX, R11 + ADCQ DX, CX + + // | j1 + + // | w4 @ R12 + MOVQ ·q+8(SB), AX + MULQ DI + ADDQ AX, R12 + ADCQ $0x00, DX + ADDQ CX, R12 + MOVQ $0x00, CX + ADCQ DX, CX + + // | j2 + + // | w5 @ R13 + MOVQ ·q+16(SB), AX + MULQ DI + ADDQ AX, R13 + ADCQ $0x00, DX + ADDQ CX, R13 + MOVQ $0x00, CX + ADCQ DX, CX + + // | j3 + + // | w6 @ R14 + MOVQ ·q+24(SB), AX + MULQ DI + ADDQ AX, R14 + ADCQ DX, R8 + ADDQ CX, R14 + + // | w-1 @ BX + ADCQ R8, BX + MOVQ $0x00, R8 + ADCQ $0x00, R8 + + // | + // | W montgomerry reduction ends + // | 0 - | 1 - | 2 - | 3 - + // | 4 R12 | 5 R13 | 6 R14 | 7 BX + + + // | + +/* modular reduction */ + + MOVQ R12, SI + SUBQ ·q+0(SB), SI + MOVQ R13, R9 + SBBQ ·q+8(SB), R9 + MOVQ R14, R10 + SBBQ ·q+16(SB), R10 + MOVQ BX, R11 + SBBQ ·q+24(SB), R11 + SBBQ $0x00, R8 + + // | + +/* out */ + + MOVQ c+0(FP), R8 + CMOVQCC SI, R12 + MOVQ R12, (R8) + CMOVQCC R9, R13 + MOVQ R13, 8(R8) + CMOVQCC R10, R14 + MOVQ R14, 16(R8) + CMOVQCC R11, BX + MOVQ BX, 24(R8) + RET +/* end */ + + +// func mulFR(c *[4]uint64, a *[4]uint64, b *[4]uint64) +TEXT ·mulADXFR(SB), NOSPLIT, $0-24 + // | + +/* inputs */ + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + XORQ AX, AX + + // | + +/* i = 0 */ + + // | a0 @ DX + MOVQ (DI), DX + + // | a0 * b0 + MULXQ (SI), CX, R8 + + // | a0 * b1 + MULXQ 8(SI), AX, R9 + ADCXQ AX, R8 + + // | a0 * b2 + MULXQ 16(SI), AX, R10 + ADCXQ AX, R9 + + // | a0 * b3 + MULXQ 24(SI), AX, R11 + ADCXQ AX, R10 + ADCQ $0x00, R11 + + // | + +/* i = 1 */ + + // | a1 @ DX + MOVQ 8(DI), DX + XORQ R12, R12 + + // | a1 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R8 + ADCXQ BX, R9 + + // | a1 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | a1 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a1 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R11 + ADOXQ R12, R12 + ADCXQ BX, R12 + + // | + +/* i = 2 */ + + // | a2 @ DX + MOVQ 16(DI), DX + XORQ R13, R13 + + // | a2 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | a2 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a2 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a2 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R12 + ADOXQ R13, R13 + ADCXQ BX, R13 + + // | + +/* i = 3 */ + + // | a3 @ DX + MOVQ 24(DI), DX + XORQ DI, DI + + // | a3 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a3 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a3 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R12 + ADCXQ BX, R13 + + // | a3 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R13 + ADOXQ BX, DI + ADCQ $0x00, DI + + // | + +/* */ + + // | + // | W + // | 0 CX | 1 R8 | 2 R9 | 3 R10 + // | 4 R11 | 5 R12 | 6 R13 | 7 DI + + + // | + // | W ready to mont + // | 0 CX | 1 R8 | 2 R9 | 3 R10 + // | 4 R11 | 5 R12 | 6 R13 | 7 DI + + + // | + +/* montgomery reduction */ + + // | clear flags + XORQ AX, AX + + // | + +/* i = 0 */ + + // | + // | W + // | 0 CX | 1 R8 | 2 R9 | 3 R10 + // | 4 R11 | 5 R12 | 6 R13 | 7 DI + + + // | | u0 = w0 * inp + MOVQ CX, DX + MULXQ ·qinp+0(SB), DX, BX + + // | + +/* */ + + // | j0 + + // | w0 @ CX + MULXQ ·q+0(SB), AX, BX + ADOXQ AX, CX + ADCXQ BX, R8 + + // | j1 + + // | w1 @ R8 + MULXQ ·q+8(SB), AX, BX + ADOXQ AX, R8 + ADCXQ BX, R9 + + // | j2 + + // | w2 @ R9 + MULXQ ·q+16(SB), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | j3 + + // | w3 @ R10 + MULXQ ·q+24(SB), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + ADOXQ CX, R11 + ADCXQ CX, CX + MOVQ $0x00, AX + ADOXQ AX, CX + + // | clear flags + XORQ AX, AX + + // | + +/* i = 1 */ + + // | + // | W + // | 0 - | 1 R8 | 2 R9 | 3 R10 + // | 4 R11 | 5 R12 | 6 R13 | 7 DI + + + // | | u1 = w1 * inp + MOVQ R8, DX + MULXQ ·qinp+0(SB), DX, BX + + // | + +/* */ + + // | j0 + + // | w1 @ R8 + MULXQ ·q+0(SB), AX, BX + ADOXQ AX, R8 + ADCXQ BX, R9 + + // | j1 + + // | w2 @ R9 + MULXQ ·q+8(SB), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | j2 + + // | w3 @ R10 + MULXQ ·q+16(SB), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | j3 + + // | w4 @ R11 + MULXQ ·q+24(SB), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + ADOXQ CX, R12 + ADCXQ R8, R8 + MOVQ $0x00, AX + ADOXQ AX, R8 + + // | clear flags + XORQ AX, AX + + // | + +/* i = 2 */ + + // | + // | W + // | 0 - | 1 - | 2 R9 | 3 R10 + // | 4 R11 | 5 R12 | 6 R13 | 7 DI + + + // | | u2 = w2 * inp + MOVQ R9, DX + MULXQ ·qinp+0(SB), DX, BX + + // | + +/* */ + + // | j0 + + // | w2 @ R9 + MULXQ ·q+0(SB), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | j1 + + // | w3 @ R10 + MULXQ ·q+8(SB), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | j2 + + // | w4 @ R11 + MULXQ ·q+16(SB), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | j3 + + // | w5 @ R12 + MULXQ ·q+24(SB), AX, BX + ADOXQ AX, R12 + ADCXQ BX, R13 + ADOXQ R8, R13 + ADCXQ R9, R9 + MOVQ $0x00, AX + ADOXQ AX, R9 + + // | clear flags + XORQ AX, AX + + // | + +/* i = 3 */ + + // | + // | W + // | 0 - | 1 - | 2 - | 3 R10 + // | 4 R11 | 5 R12 | 6 R13 | 7 DI + + + // | | u3 = w3 * inp + MOVQ R10, DX + MULXQ ·qinp+0(SB), DX, BX + + // | + +/* */ + + // | j0 + + // | w3 @ R10 + MULXQ ·q+0(SB), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | j1 + + // | w4 @ R11 + MULXQ ·q+8(SB), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | j2 + + // | w5 @ R12 + MULXQ ·q+16(SB), AX, BX + ADOXQ AX, R12 + ADCXQ BX, R13 + + // | j3 + + // | w6 @ R13 + MULXQ ·q+24(SB), AX, BX + ADOXQ AX, R13 + ADCXQ BX, DI + ADOXQ R9, DI + ADCXQ R10, R10 + MOVQ $0x00, AX + ADOXQ AX, R10 + + // | + // | W montgomery reduction ends + // | 0 - | 1 - | 2 - | 3 - + // | 4 R11 | 5 R12 | 6 R13 | 7 DI + + + // | + +/* modular reduction */ + + MOVQ R11, CX + SUBQ ·q+0(SB), CX + MOVQ R12, AX + SBBQ ·q+8(SB), AX + MOVQ R13, BX + SBBQ ·q+16(SB), BX + MOVQ DI, SI + SBBQ ·q+24(SB), SI + SBBQ $0x00, R10 + + // | + +/* out */ + + MOVQ c+0(FP), R10 + CMOVQCC CX, R11 + MOVQ R11, (R10) + CMOVQCC AX, R12 + MOVQ R12, 8(R10) + CMOVQCC BX, R13 + MOVQ R13, 16(R10) + CMOVQCC SI, DI + MOVQ DI, 24(R10) + RET +/* end */ + + +TEXT ·waddFR(SB), NOSPLIT, $0-16 + // | + MOVQ a+0(FP), DI + MOVQ b+8(FP), SI + + // | + MOVQ (DI), R8 + MOVQ 8(DI), R9 + MOVQ 16(DI), R10 + MOVQ 24(DI), R11 + MOVQ 32(DI), R12 + MOVQ 40(DI), R13 + MOVQ 48(DI), R14 + MOVQ 56(DI), R15 + + // | + ADDQ (SI), R8 + ADCQ 8(SI), R9 + ADCQ 16(SI), R10 + ADCQ 24(SI), R11 + ADCQ 32(SI), R12 + ADCQ 40(SI), R13 + ADCQ 48(SI), R14 + ADCQ 56(SI), R15 + + // | + MOVQ R8, (DI) + MOVQ R9, 8(DI) + MOVQ R10, 16(DI) + MOVQ R11, 24(DI) + MOVQ R12, 32(DI) + MOVQ R13, 40(DI) + MOVQ R14, 48(DI) + MOVQ R15, 56(DI) + RET +/* end */ + +// func wmulNoADXFR(c *[8]uint64, a *[4]uint64, b *[4]uint64) +TEXT ·wmulNoADXFR(SB), NOSPLIT, $0-24 + // | + +/* inputs */ + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + MOVQ $0x00, R10 + MOVQ $0x00, R11 + MOVQ $0x00, R12 + MOVQ $0x00, R13 + MOVQ $0x00, R14 + + // | + +/* i = 0 */ + + // | a0 @ CX + MOVQ (DI), CX + + // | a0 * b0 + MOVQ (SI), AX + MULQ CX + MOVQ AX, R8 + MOVQ DX, R9 + + // | a0 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + + // | a0 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + + // | a0 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + + // | + +/* i = 1 */ + + // | a1 @ CX + MOVQ 8(DI), CX + MOVQ $0x00, BX + + // | a1 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R9 + ADCQ DX, R10 + ADCQ $0x00, R11 + ADCQ $0x00, BX + + // | a1 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ BX, R12 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a1 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + + // | a1 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + + // | + +/* i = 2 */ + + // | a2 @ CX + MOVQ 16(DI), CX + MOVQ $0x00, BX + + // | a2 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R10 + ADCQ DX, R11 + ADCQ $0x00, R12 + ADCQ $0x00, BX + + // | a2 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ BX, R13 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a2 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + + // | a2 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + + // | + +/* i = 3 */ + + // | a3 @ CX + MOVQ 24(DI), CX + MOVQ $0x00, BX + + // | a3 * b0 + MOVQ (SI), AX + MULQ CX + ADDQ AX, R11 + ADCQ DX, R12 + ADCQ $0x00, R13 + ADCQ $0x00, BX + + // | a3 * b1 + MOVQ 8(SI), AX + MULQ CX + ADDQ AX, R12 + ADCQ DX, R13 + ADCQ BX, R14 + MOVQ $0x00, BX + ADCQ $0x00, BX + + // | a3 * b2 + MOVQ 16(SI), AX + MULQ CX + ADDQ AX, R13 + ADCQ DX, R14 + ADCQ $0x00, BX + + // | a3 * b3 + MOVQ 24(SI), AX + MULQ CX + ADDQ AX, R14 + ADCQ DX, BX + + // | + +/* */ + + // | + // | W + // | 0 R8 | 1 R9 | 2 R10 | 3 R11 + // | 4 R12 | 5 R13 | 6 R14 | 7 BX + + MOVQ c+0(FP), AX + MOVQ R8, (AX) + MOVQ R9, 8(AX) + MOVQ R10, 16(AX) + MOVQ R11, 24(AX) + MOVQ R12, 32(AX) + MOVQ R13, 40(AX) + MOVQ R14, 48(AX) + MOVQ BX, 56(AX) + + RET +/* end */ + + +// func wmulADXFR(c *[8]uint64, a *[4]uint64, b *[4]uint64) +TEXT ·wmulADXFR(SB), NOSPLIT, $0-24 + // | + +/* inputs */ + + MOVQ a+8(FP), DI + MOVQ b+16(FP), SI + XORQ AX, AX + + // | + +/* i = 0 */ + + // | a0 @ DX + MOVQ (DI), DX + + // | a0 * b0 + MULXQ (SI), CX, R8 + + // | a0 * b1 + MULXQ 8(SI), AX, R9 + ADCXQ AX, R8 + + // | a0 * b2 + MULXQ 16(SI), AX, R10 + ADCXQ AX, R9 + + // | a0 * b3 + MULXQ 24(SI), AX, R11 + ADCXQ AX, R10 + ADCQ $0x00, R11 + + // | + +/* i = 1 */ + + // | a1 @ DX + MOVQ 8(DI), DX + XORQ R12, R12 + + // | a1 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R8 + ADCXQ BX, R9 + + // | a1 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | a1 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a1 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R11 + ADOXQ R12, R12 + ADCXQ BX, R12 + + // | + +/* i = 2 */ + + // | a2 @ DX + MOVQ 16(DI), DX + XORQ R13, R13 + + // | a2 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R9 + ADCXQ BX, R10 + + // | a2 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a2 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a2 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R12 + ADOXQ R13, R13 + ADCXQ BX, R13 + + // | + +/* i = 3 */ + + // | a3 @ DX + MOVQ 24(DI), DX + XORQ DI, DI + + // | a3 * b0 + MULXQ (SI), AX, BX + ADOXQ AX, R10 + ADCXQ BX, R11 + + // | a3 * b1 + MULXQ 8(SI), AX, BX + ADOXQ AX, R11 + ADCXQ BX, R12 + + // | a3 * b2 + MULXQ 16(SI), AX, BX + ADOXQ AX, R12 + ADCXQ BX, R13 + + // | a3 * b3 + MULXQ 24(SI), AX, BX + ADOXQ AX, R13 + ADOXQ BX, DI + ADCQ $0x00, DI + + // | + +/* */ + + // | + // | W + // | 0 CX | 1 R8 | 2 R9 | 3 R10 + // | 4 R11 | 5 R12 | 6 R13 | 7 DI + + + MOVQ c+0(FP), AX + MOVQ CX, (AX) + MOVQ R8, 8(AX) + MOVQ R9, 16(AX) + MOVQ R10, 24(AX) + MOVQ R11, 32(AX) + MOVQ R12, 40(AX) + MOVQ R13, 48(AX) + MOVQ DI, 56(AX) + + RET + +/* end */ \ No newline at end of file diff --git a/crypto/bls12381/fr_fallback.go b/crypto/bls12381/fr_fallback.go new file mode 100644 index 0000000000..236d3cf3ae --- /dev/null +++ b/crypto/bls12381/fr_fallback.go @@ -0,0 +1,383 @@ +// +build !amd64 generic + +// Copyright 2020 ConsenSys Software Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by goff (v0.3.5) DO NOT EDIT + +// /!\ WARNING /!\ +// this code has not been audited and is provided as-is. In particular, +// there is no security guarantees such as constant time implementation +// or side-channel attack resistance +// /!\ WARNING /!\ + +package bls12381 + +import "math/bits" + +func addFR(z, x, y *Fr) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], y[0], 0) + z[1], carry = bits.Add64(x[1], y[1], carry) + z[2], carry = bits.Add64(x[2], y[2], carry) + z[3], _ = bits.Add64(x[3], y[3], carry) + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[3] < 8353516859464449352 || (z[3] == 8353516859464449352 && (z[2] < 3691218898639771653 || (z[2] == 3691218898639771653 && (z[1] < 6034159408538082302 || (z[1] == 6034159408538082302 && (z[0] < 18446744069414584321))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 18446744069414584321, 0) + z[1], b = bits.Sub64(z[1], 6034159408538082302, b) + z[2], b = bits.Sub64(z[2], 3691218898639771653, b) + z[3], _ = bits.Sub64(z[3], 8353516859464449352, b) + } +} + +func laddAssignFR(z, y *Fr) { + var carry uint64 + + z[0], carry = bits.Add64(z[0], y[0], 0) + z[1], carry = bits.Add64(z[1], y[1], carry) + z[2], carry = bits.Add64(z[2], y[2], carry) + z[3], _ = bits.Add64(z[3], y[3], carry) +} + +func doubleFR(z, x *Fr) { + var carry uint64 + + z[0], carry = bits.Add64(x[0], x[0], 0) + z[1], carry = bits.Add64(x[1], x[1], carry) + z[2], carry = bits.Add64(x[2], x[2], carry) + z[3], _ = bits.Add64(x[3], x[3], carry) + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[3] < 8353516859464449352 || (z[3] == 8353516859464449352 && (z[2] < 3691218898639771653 || (z[2] == 3691218898639771653 && (z[1] < 6034159408538082302 || (z[1] == 6034159408538082302 && (z[0] < 18446744069414584321))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 18446744069414584321, 0) + z[1], b = bits.Sub64(z[1], 6034159408538082302, b) + z[2], b = bits.Sub64(z[2], 3691218898639771653, b) + z[3], _ = bits.Sub64(z[3], 8353516859464449352, b) + } +} + +func subFR(z, x, y *Fr) { + var b uint64 + z[0], b = bits.Sub64(x[0], y[0], 0) + z[1], b = bits.Sub64(x[1], y[1], b) + z[2], b = bits.Sub64(x[2], y[2], b) + z[3], b = bits.Sub64(x[3], y[3], b) + if b != 0 { + var c uint64 + z[0], c = bits.Add64(z[0], 18446744069414584321, 0) + z[1], c = bits.Add64(z[1], 6034159408538082302, c) + z[2], c = bits.Add64(z[2], 3691218898639771653, c) + z[3], _ = bits.Add64(z[3], 8353516859464449352, c) + } +} + +func lsubAssignFR(z, y *Fr) { + var b uint64 + z[0], b = bits.Sub64(z[0], y[0], 0) + z[1], b = bits.Sub64(z[1], y[1], b) + z[2], b = bits.Sub64(z[2], y[2], b) + z[3], b = bits.Sub64(z[3], y[3], b) +} + +func negFR(z, x *Fr) { + if x.IsZero() { + z.Zero() + return + } + var borrow uint64 + z[0], borrow = bits.Sub64(18446744069414584321, x[0], 0) + z[1], borrow = bits.Sub64(6034159408538082302, x[1], borrow) + z[2], borrow = bits.Sub64(3691218898639771653, x[2], borrow) + z[3], _ = bits.Sub64(8353516859464449352, x[3], borrow) +} + +func mulFR(z, x, y *Fr) { + + var t [4]uint64 + var c [3]uint64 + { + // round 0 + v := x[0] + c[1], c[0] = bits.Mul64(v, y[0]) + m := c[0] * 18446744069414584319 + c[2] = madd0(m, 18446744069414584321, c[0]) + c[1], c[0] = madd1(v, y[1], c[1]) + c[2], t[0] = madd2(m, 6034159408538082302, c[2], c[0]) + c[1], c[0] = madd1(v, y[2], c[1]) + c[2], t[1] = madd2(m, 3691218898639771653, c[2], c[0]) + c[1], c[0] = madd1(v, y[3], c[1]) + t[3], t[2] = madd3(m, 8353516859464449352, c[0], c[2], c[1]) + } + { + // round 1 + v := x[1] + c[1], c[0] = madd1(v, y[0], t[0]) + m := c[0] * 18446744069414584319 + c[2] = madd0(m, 18446744069414584321, c[0]) + c[1], c[0] = madd2(v, y[1], c[1], t[1]) + c[2], t[0] = madd2(m, 6034159408538082302, c[2], c[0]) + c[1], c[0] = madd2(v, y[2], c[1], t[2]) + c[2], t[1] = madd2(m, 3691218898639771653, c[2], c[0]) + c[1], c[0] = madd2(v, y[3], c[1], t[3]) + t[3], t[2] = madd3(m, 8353516859464449352, c[0], c[2], c[1]) + } + { + // round 2 + v := x[2] + c[1], c[0] = madd1(v, y[0], t[0]) + m := c[0] * 18446744069414584319 + c[2] = madd0(m, 18446744069414584321, c[0]) + c[1], c[0] = madd2(v, y[1], c[1], t[1]) + c[2], t[0] = madd2(m, 6034159408538082302, c[2], c[0]) + c[1], c[0] = madd2(v, y[2], c[1], t[2]) + c[2], t[1] = madd2(m, 3691218898639771653, c[2], c[0]) + c[1], c[0] = madd2(v, y[3], c[1], t[3]) + t[3], t[2] = madd3(m, 8353516859464449352, c[0], c[2], c[1]) + } + { + // round 3 + v := x[3] + c[1], c[0] = madd1(v, y[0], t[0]) + m := c[0] * 18446744069414584319 + c[2] = madd0(m, 18446744069414584321, c[0]) + c[1], c[0] = madd2(v, y[1], c[1], t[1]) + c[2], z[0] = madd2(m, 6034159408538082302, c[2], c[0]) + c[1], c[0] = madd2(v, y[2], c[1], t[2]) + c[2], z[1] = madd2(m, 3691218898639771653, c[2], c[0]) + c[1], c[0] = madd2(v, y[3], c[1], t[3]) + z[3], z[2] = madd3(m, 8353516859464449352, c[0], c[2], c[1]) + } + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[3] < 8353516859464449352 || (z[3] == 8353516859464449352 && (z[2] < 3691218898639771653 || (z[2] == 3691218898639771653 && (z[1] < 6034159408538082302 || (z[1] == 6034159408538082302 && (z[0] < 18446744069414584321))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 18446744069414584321, 0) + z[1], b = bits.Sub64(z[1], 6034159408538082302, b) + z[2], b = bits.Sub64(z[2], 3691218898639771653, b) + z[3], _ = bits.Sub64(z[3], 8353516859464449352, b) + } +} + +func squareFR(z, x *Fr) { + + var t [4]uint64 + var c [3]uint64 + { + // round 0 + v := x[0] + c[1], c[0] = bits.Mul64(v, x[0]) + m := c[0] * 18446744069414584319 + c[2] = madd0(m, 18446744069414584321, c[0]) + c[1], c[0] = madd1(v, x[1], c[1]) + c[2], t[0] = madd2(m, 6034159408538082302, c[2], c[0]) + c[1], c[0] = madd1(v, x[2], c[1]) + c[2], t[1] = madd2(m, 3691218898639771653, c[2], c[0]) + c[1], c[0] = madd1(v, x[3], c[1]) + t[3], t[2] = madd3(m, 8353516859464449352, c[0], c[2], c[1]) + } + { + // round 1 + v := x[1] + c[1], c[0] = madd1(v, x[0], t[0]) + m := c[0] * 18446744069414584319 + c[2] = madd0(m, 18446744069414584321, c[0]) + c[1], c[0] = madd2(v, x[1], c[1], t[1]) + c[2], t[0] = madd2(m, 6034159408538082302, c[2], c[0]) + c[1], c[0] = madd2(v, x[2], c[1], t[2]) + c[2], t[1] = madd2(m, 3691218898639771653, c[2], c[0]) + c[1], c[0] = madd2(v, x[3], c[1], t[3]) + t[3], t[2] = madd3(m, 8353516859464449352, c[0], c[2], c[1]) + } + { + // round 2 + v := x[2] + c[1], c[0] = madd1(v, x[0], t[0]) + m := c[0] * 18446744069414584319 + c[2] = madd0(m, 18446744069414584321, c[0]) + c[1], c[0] = madd2(v, x[1], c[1], t[1]) + c[2], t[0] = madd2(m, 6034159408538082302, c[2], c[0]) + c[1], c[0] = madd2(v, x[2], c[1], t[2]) + c[2], t[1] = madd2(m, 3691218898639771653, c[2], c[0]) + c[1], c[0] = madd2(v, x[3], c[1], t[3]) + t[3], t[2] = madd3(m, 8353516859464449352, c[0], c[2], c[1]) + } + { + // round 3 + v := x[3] + c[1], c[0] = madd1(v, x[0], t[0]) + m := c[0] * 18446744069414584319 + c[2] = madd0(m, 18446744069414584321, c[0]) + c[1], c[0] = madd2(v, x[1], c[1], t[1]) + c[2], z[0] = madd2(m, 6034159408538082302, c[2], c[0]) + c[1], c[0] = madd2(v, x[2], c[1], t[2]) + c[2], z[1] = madd2(m, 3691218898639771653, c[2], c[0]) + c[1], c[0] = madd2(v, x[3], c[1], t[3]) + z[3], z[2] = madd3(m, 8353516859464449352, c[0], c[2], c[1]) + } + + // if z > q --> z -= q + // note: this is NOT constant time + if !(z[3] < 8353516859464449352 || (z[3] == 8353516859464449352 && (z[2] < 3691218898639771653 || (z[2] == 3691218898639771653 && (z[1] < 6034159408538082302 || (z[1] == 6034159408538082302 && (z[0] < 18446744069414584321))))))) { + var b uint64 + z[0], b = bits.Sub64(z[0], 18446744069414584321, 0) + z[1], b = bits.Sub64(z[1], 6034159408538082302, b) + z[2], b = bits.Sub64(z[2], 3691218898639771653, b) + z[3], _ = bits.Sub64(z[3], 8353516859464449352, b) + } +} + +func waddFR(z, y *wideFr) { + var carry uint64 + z[0], carry = bits.Add64(z[0], y[0], 0) + z[1], carry = bits.Add64(z[1], y[1], carry) + z[2], carry = bits.Add64(z[2], y[2], carry) + z[3], carry = bits.Add64(z[3], y[3], carry) + z[4], carry = bits.Add64(z[4], y[4], carry) + z[5], carry = bits.Add64(z[5], y[5], carry) + z[6], carry = bits.Add64(z[6], y[6], carry) + z[7], _ = bits.Add64(z[7], y[7], carry) +} + +// We applied custom multiplication since goff does generate multiplication code nested with reduction +func wmulFR(w *wideFr, a, b *Fr) { + // Handbook of Applied Cryptography + // Hankerson, Menezes, Vanstone + // 14.12 Algorithm Multiple-precision multiplication + + var w0, w1, w2, w3, w4, w5, w6, w7 uint64 + var a0 = a[0] + var a1 = a[1] + var a2 = a[2] + var a3 = a[3] + var b0 = b[0] + var b1 = b[1] + var b2 = b[2] + var b3 = b[3] + var u, v, c, t uint64 + + // i = 0, j = 0 + c, w0 = bits.Mul64(a0, b0) + + // i = 0, j = 1 + u, v = bits.Mul64(a1, b0) + w1 = v + c + c = u + (v&c|(v|c)&^w1)>>63 + + // i = 0, j = 2 + u, v = bits.Mul64(a2, b0) + w2 = v + c + c = u + (v&c|(v|c)&^w2)>>63 + + // i = 0, j = 3 + u, v = bits.Mul64(a3, b0) + w3 = v + c + w4 = u + (v&c|(v|c)&^w3)>>63 + + // i = 1, j = 0 + c, v = bits.Mul64(a0, b1) + t = v + w1 + c += (v&w1 | (v|w1)&^t) >> 63 + w1 = t + + // i = 1, j = 1 + u, v = bits.Mul64(a1, b1) + t = v + w2 + u += (v&w2 | (v|w2)&^t) >> 63 + w2 = t + c + c = u + (t&c|(t|c)&^w2)>>63 + + // i = 1, j = 2 + u, v = bits.Mul64(a2, b1) + t = v + w3 + u += (v&w3 | (v|w3)&^t) >> 63 + w3 = t + c + c = u + (t&c|(t|c)&^w3)>>63 + + // i = 1, j = 3 + u, v = bits.Mul64(a3, b1) + t = v + w4 + u += (v&w4 | (v|w4)&^t) >> 63 + w4 = t + c + w5 = u + (t&c|(t|c)&^w4)>>63 + + // i = 2, j = 0 + c, v = bits.Mul64(a0, b2) + t = v + w2 + c += (v&w2 | (v|w2)&^t) >> 63 + w2 = t + + // i = 2, j = 1 + u, v = bits.Mul64(a1, b2) + t = v + w3 + u += (v&w3 | (v|w3)&^t) >> 63 + w3 = t + c + c = u + (t&c|(t|c)&^w3)>>63 + + // i = 2, j = 2 + u, v = bits.Mul64(a2, b2) + t = v + w4 + u += (v&w4 | (v|w4)&^t) >> 63 + w4 = t + c + c = u + (t&c|(t|c)&^w4)>>63 + + // i = 2, j = 3 + u, v = bits.Mul64(a3, b2) + t = v + w5 + u += (v&w5 | (v|w5)&^t) >> 63 + w5 = t + c + w6 = u + (t&c|(t|c)&^w5)>>63 + + // i = 3, j = 0 + c, v = bits.Mul64(a0, b3) + t = v + w3 + c += (v&w3 | (v|w3)&^t) >> 63 + w3 = t + + // i = 3, j = 1 + u, v = bits.Mul64(a1, b3) + t = v + w4 + u += (v&w4 | (v|w4)&^t) >> 63 + w4 = t + c + c = u + (t&c|(t|c)&^w4)>>63 + + // i = 3, j = 2 + u, v = bits.Mul64(a2, b3) + t = v + w5 + u += (v&w5 | (v|w5)&^t) >> 63 + w5 = t + c + c = u + (t&c|(t|c)&^w5)>>63 + + // i = 3, j = 3 + u, v = bits.Mul64(a3, b3) + t = v + w6 + u += (v&w6 | (v|w6)&^t) >> 63 + w6 = t + c + w7 = u + (t&c|(t|c)&^w6)>>63 + + w[0] = w0 + w[1] = w1 + w[2] = w2 + w[3] = w3 + w[4] = w4 + w[5] = w5 + w[6] = w6 + w[7] = w7 +} diff --git a/crypto/bls12381/fr_test.go b/crypto/bls12381/fr_test.go new file mode 100644 index 0000000000..20641f2880 --- /dev/null +++ b/crypto/bls12381/fr_test.go @@ -0,0 +1,417 @@ +package bls12381 + +import ( + "bytes" + "crypto/rand" + "math/big" + "testing" +) + +func TestScalarField(t *testing.T) { + r := new(Fr).Set(qr1) + r.fromMont() + if r[0] != 1 && r[1] != 0 && r[2] != 0 && r[3] != 0 { + t.Fatal("bad r value") + } + r.Set(qr2) + r.fromMont() + r.fromMont() + if r[0] != 1 && r[1] != 0 && r[2] != 0 && r[3] != 0 { + t.Fatal("bad r2 value") + } + r = &Fr{1} + r.toMont() + if !r.Equal(qr1) { + t.Fatal("mont transformaition failed") + } +} + +func TestFrSerialization(t *testing.T) { + in := make([]byte, frByteSize) + + e := new(Fr).FromBytes(in) + if !e.IsZero() { + t.Fatal("serialization failed, from bytes zero") + } + if !bytes.Equal(in, e.ToBytes()) { + t.Fatal("serialization failed, to bytes zero") + } + + e = new(Fr).RedFromBytes(in) + if !e.IsZero() { + t.Fatal("serialization failed, from bytes zero, reduced") + } + if !bytes.Equal(in, e.RedToBytes()) { + t.Fatal("serialization failed, to bytes zero, reduced") + } + + a, err := new(Fr).Rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + b := new(Fr) + b.fromBytes(a.bytes()) + if !a.Equal(b) { + t.Fatal("serialization failed, set bytes") + } + + b = new(Fr).FromBytes(a.ToBytes()) + if !a.Equal(b) { + t.Fatal("serialization failed, from/to bytes") + } + + b = new(Fr).RedFromBytes(a.RedToBytes()) + if !a.Equal(b) { + t.Fatal("serialization failed, from/to bytes, reduced") + } +} + +func TestFrSliceUint(t *testing.T) { + s, err := new(Fr).Rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + sBig := s.ToBig() + for offset := 0; offset < 260; offset++ { + a0 := new(big.Int).Rsh(sBig, uint(offset)).Uint64() + a1 := s.sliceUint64(offset) + if a0 != a1 { + t.Fatal("uint slice failed", offset) + } + } +} + +func TestFrBitTest(t *testing.T) { + s, err := new(Fr).Rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + sBig := s.ToBig() + for i := 0; i < 260; i++ { + a0 := sBig.Bit(i) == 1 + a1 := s.Bit(i) + if a0 != a1 { + t.Fatal("bit test failed", i) + } + } +} + +func TestFrBitShift(t *testing.T) { + a, _ := new(Fr).Rand(rand.Reader) + b := new(Fr).Set(a) + b.mul2() + b.div2() + if !b.Equal(a) { + t.Fatal("mul2 div2 failed") + } + a, _ = new(Fr).Rand(rand.Reader) + a[0] = a[0] & 0xfffffffffffffffe + b.Set(a) + b.div2() + b.mul2() + if !b.Equal(a) { + t.Fatal("mul2 div2 failed") + } +} + +func TestFrAdditionCrossAgainstBigInt(t *testing.T) { + for i := 0; i < fuz; i++ { + a, _ := new(Fr).Rand(rand.Reader) + b, _ := new(Fr).Rand(rand.Reader) + c := new(Fr) + bigA := a.ToBig() + bigB := b.ToBig() + bigC := new(big.Int) + c.Add(a, b) + out1 := c.ToBytes() + out2 := padBytes(bigC.Add(bigA, bigB).Mod(bigC, qBig).Bytes(), frByteSize) + if !bytes.Equal(out1, out2) { + t.Fatal("cross test against big.Int is failed, add") + } + c.Double(a) + out1 = c.ToBytes() + out2 = padBytes(bigC.Add(bigA, bigA).Mod(bigC, qBig).Bytes(), frByteSize) + if !bytes.Equal(out1, out2) { + t.Fatal("cross test against big.Int is failed, double") + } + c.Sub(a, b) + out1 = c.ToBytes() + out2 = padBytes(bigC.Sub(bigA, bigB).Mod(bigC, qBig).Bytes(), frByteSize) + if !bytes.Equal(out1, out2) { + t.Fatal("cross test against big.Int is failed, sub") + } + c.Neg(a) + out1 = c.ToBytes() + out2 = padBytes(bigC.Neg(bigA).Mod(bigC, qBig).Bytes(), frByteSize) + if !bytes.Equal(out1, out2) { + t.Fatal("cross test against big.Int is failed, neg") + } + } +} + +func TestFrAdditionProperties(t *testing.T) { + for i := 0; i < fuz; i++ { + zero := new(Fr) + a, _ := new(Fr).Rand(rand.Reader) + b, _ := new(Fr).Rand(rand.Reader) + c1, c2 := new(Fr), new(Fr) + c1.Add(a, zero) + if !c1.Equal(a) { + t.Fatal("a + 0 == a") + } + c1.Sub(a, zero) + if !c1.Equal(a) { + t.Fatal("a - 0 == a") + } + c1.Double(zero) + if !c1.Equal(zero) { + t.Fatal("2 * 0 == 0") + } + c1.Neg(zero) + if !c1.Equal(zero) { + t.Fatal("-0 == 0") + } + c1.Sub(zero, a) + c2.Neg(a) + if !c1.Equal(c2) { + t.Fatal("0-a == -a") + } + c1.Double(a) + c2.Add(a, a) + if !c1.Equal(c2) { + t.Fatal("2 * a == a + a") + } + c1.Add(a, b) + c2.Add(b, a) + if !c1.Equal(c2) { + t.Fatal("a + b = b + a") + } + c1.Sub(a, b) + c2.Sub(b, a) + c2.Neg(c2) + if !c1.Equal(c2) { + t.Fatal("a - b = - ( b - a )") + } + c0, _ := new(Fr).Rand(rand.Reader) + c1.Add(a, b) + c1.Add(c1, c0) + c2.Add(a, c0) + c2.Add(c2, b) + if !c1.Equal(c2) { + t.Fatal("(a + b) + c == (a + c ) + b") + } + c1.Sub(a, b) + c1.Sub(c1, c0) + c2.Sub(a, c0) + c2.Sub(c2, b) + if !c1.Equal(c2) { + t.Fatal("(a - b) - c == (a - c ) -b") + } + } +} + +func TestFrMultiplicationCrossAgainstBigInt(t *testing.T) { + for i := 0; i < fuz; i++ { + a, _ := new(Fr).Rand(rand.Reader) + b, _ := new(Fr).Rand(rand.Reader) + c := new(Fr) + bigA := a.ToBig() + bigB := b.ToBig() + bigC := new(big.Int) + c.Mul(a, b) + out1 := c.ToBytes() + out2 := padBytes(bigC.Mul(bigA, bigB).Mod(bigC, qBig).Bytes(), frByteSize) + if !bytes.Equal(out1, out2) { + t.Fatal("cross test against big.Int is failed") + } + } +} + +func TestFrMultiplicationCrossAgainstBigIntReduced(t *testing.T) { + for i := 0; i < fuz; i++ { + a, _ := new(Fr).Rand(rand.Reader) + b, _ := new(Fr).Rand(rand.Reader) + c := new(Fr) + bigA := a.RedToBig() + bigB := b.RedToBig() + bigC := new(big.Int) + c.RedMul(a, b) + out1 := c.RedToBytes() + out2 := padBytes(bigC.Mul(bigA, bigB).Mod(bigC, qBig).Bytes(), frByteSize) + if !bytes.Equal(out1, out2) { + t.Fatal("cross test against big.Int is failed, reduced") + } + } +} + +func TestFrMultiplicationProperties(t *testing.T) { + for i := 0; i < fuz; i++ { + a, _ := new(Fr).Rand(rand.Reader) + b, _ := new(Fr).Rand(rand.Reader) + zero, one := new(Fr).Zero(), new(Fr).One() + c1, c2 := new(Fr), new(Fr) + c1.Mul(a, zero) + if !c1.Equal(zero) { + t.Fatal("a * 0 == 0") + } + c1.Mul(a, one) + if !c1.Equal(a) { + t.Fatal("a * 1 == a") + } + c1.Mul(a, b) + c2.Mul(b, a) + if !c1.Equal(c2) { + t.Fatal("a * b == b * a") + } + c0, _ := new(Fr).Rand(rand.Reader) + c1.Mul(a, b) + c1.Mul(c1, c0) + c2.Mul(c0, b) + c2.Mul(c2, a) + if !c1.Equal(c2) { + t.Fatal("(a * b) * c == (a * c) * b") + } + a.Square(zero) + if !a.Equal(zero) { + t.Fatal("0^2 == 0") + } + a.Square(one) + if !a.Equal(one) { + t.Fatal("1^2 == 1") + } + _, _ = a.Rand(rand.Reader) + c1.Square(a) + c2.Mul(a, a) + if !c1.Equal(c1) { + t.Fatal("a^2 == a*a") + } + } +} + +func TestFrMultiplicationPropertiesReduced(t *testing.T) { + for i := 0; i < fuz; i++ { + a, _ := new(Fr).Rand(rand.Reader) + b, _ := new(Fr).Rand(rand.Reader) + zero, one := new(Fr).Zero(), new(Fr).RedOne() + c1, c2 := new(Fr), new(Fr) + c1.RedMul(a, zero) + if !c1.Equal(zero) { + t.Fatal("a * 0 == 0") + } + c1.RedMul(a, one) + if !c1.Equal(a) { + t.Fatal("a * 1 == a") + } + c1.RedMul(a, b) + c2.RedMul(b, a) + if !c1.Equal(c2) { + t.Fatal("a * b == b * a") + } + c0, _ := new(Fr).Rand(rand.Reader) + c1.RedMul(a, b) + c1.RedMul(c1, c0) + c2.RedMul(c0, b) + c2.RedMul(c2, a) + if !c1.Equal(c2) { + t.Fatal("(a * b) * c == (a * c) * b") + } + a.RedSquare(zero) + if !a.Equal(zero) { + t.Fatal("0^2 == 0") + } + a.RedSquare(one) + if !a.Equal(one) { + t.Fatal("1^2 == 1") + } + _, _ = a.Rand(rand.Reader) + c1.RedSquare(a) + c2.RedMul(a, a) + if !c1.Equal(c1) { + t.Fatal("a^2 == a*a") + } + } +} + +func TestFrExponentiation(t *testing.T) { + for i := 0; i < fuz; i++ { + a, _ := new(Fr).Rand(rand.Reader) + u := new(Fr) + u.Exp(a, big.NewInt(0)) + if !u.IsOne() { + t.Fatal("a^0 == 1") + } + u.Exp(a, big.NewInt(1)) + if !u.Equal(a) { + t.Fatal("a^1 == a") + } + v := new(Fr) + u.Mul(a, a) + u.Mul(u, u) + u.Mul(u, u) + v.Exp(a, big.NewInt(8)) + if !u.Equal(v) { + t.Fatal("((a^2)^2)^2 == a^8") + } + u.Exp(a, qBig) + if !u.Equal(a) { + t.Fatal("a^p == a") + } + qMinus1 := new(big.Int).Sub(qBig, big.NewInt(1)) + u.Exp(a, qMinus1) + if !u.IsOne() { + t.Fatal("a^(p-1) == 1") + } + } +} + +func TestFrInversion(t *testing.T) { + for i := 0; i < fuz; i++ { + u := new(Fr) + zero, one := new(Fr).Zero(), new(Fr).One() + u.Inverse(zero) + if !u.Equal(zero) { + t.Fatal("(0^-1) == 0)") + } + u.Inverse(one) + if !u.IsOne() { + t.Fatal("(1^-1) == 1)") + } + a, _ := new(Fr).Rand(rand.Reader) + u.Inverse(a) + u.Mul(u, a) + if !u.IsOne() { + t.Fatal("a * a^-1 == 1") + } + v := new(Fr) + z := new(big.Int) + u.Exp(a, z.Sub(qBig, big.NewInt(2))) + v.Inverse(a) + if !v.Equal(u) { + t.Fatal("a^(p-2) == a^-1") + } + } +} + +func TestFnBatchInversion(t *testing.T) { + for i := 0; i < fuz; i++ { + zero, one := new(Fr).Zero(), new(Fr).One() + + a, _ := new(Fr).Rand(rand.Reader) + u := new(Fr) + z := new(big.Int) + u.Exp(a, z.Sub(qBig, big.NewInt(2))) + + var arr []Fr + arr = append(arr, *zero, *one, *a) + InverseBatchFr(arr) + if !arr[0].Equal(zero) { + t.Fatal("(0^-1) == 0)") + } + if !arr[1].IsOne() { + t.Fatal("(1^-1) == 1)") + } + if !arr[2].Equal(u) { + t.Fatal("a^(p-2) == a^-1") + } + } +} diff --git a/crypto/bls12381/g1.go b/crypto/bls12381/g1.go index bcb898027a..20e54ffcec 100644 --- a/crypto/bls12381/g1.go +++ b/crypto/bls12381/g1.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( @@ -22,11 +6,12 @@ import ( "math/big" ) -// PointG1 is type for point in G1. -// PointG1 is both used for Affine and Jacobian point representation. -// If z is equal to one the point is considered as in affine form. +// PointG1 is type for point in G1 and used for both Affine and Jacobian point representation. +// A point is accounted as in affine form if z is equal to one. type PointG1 [3]fe +var wnafMulWindowG1 uint = 5 + func (p *PointG1) Set(p2 *PointG1) *PointG1 { p[0].set(&p2[0]) p[1].set(&p2[1]) @@ -34,7 +19,6 @@ func (p *PointG1) Set(p2 *PointG1) *PointG1 { return p } -// Zero returns G1 point in point at infinity representation func (p *PointG1) Zero() *PointG1 { p[0].zero() p[1].one() @@ -42,6 +26,11 @@ func (p *PointG1) Zero() *PointG1 { return p } +// IsAffine checks a G1 point whether it is in affine form. +func (p *PointG1) IsAffine() bool { + return p[2].isOne() +} + type tempG1 struct { t [9]*fe } @@ -67,15 +56,141 @@ func newTempG1() tempG1 { // Q returns group order in big.Int. func (g *G1) Q() *big.Int { - return new(big.Int).Set(q) + return new(big.Int).Set(qBig) } -func (g *G1) fromBytesUnchecked(in []byte) (*PointG1, error) { - p0, err := fromBytes(in[:48]) +// FromUncompressed expects byte slice at least 96 bytes and given bytes returns a new point in G1. +// Serialization rules are in line with zcash library. See below for details. +// https://github.com/zcash/librustzcash/blob/master/pairing/src/bls12_381/README.md#serialization +// https://docs.rs/bls12_381/0.1.1/bls12_381/notes/serialization/index.html +func (g *G1) FromUncompressed(uncompressed []byte) (*PointG1, error) { + if len(uncompressed) != 2*fpByteSize { + return nil, errors.New("input string length must be equal to 96 bytes") + } + var in [2 * fpByteSize]byte + copy(in[:], uncompressed[:2*fpByteSize]) + if in[0]&(1<<7) != 0 { + return nil, errors.New("compression flag must be zero") + } + if in[0]&(1<<5) != 0 { + return nil, errors.New("sort flag must be zero") + } + if in[0]&(1<<6) != 0 { + for i, v := range in { + if (i == 0 && v != 0x40) || (i != 0 && v != 0x00) { + return nil, errors.New("input string must be zero when infinity flag is set") + } + } + return g.Zero(), nil + } + in[0] &= 0x1f + x, err := fromBytes(in[:fpByteSize]) if err != nil { return nil, err } - p1, err := fromBytes(in[48:]) + y, err := fromBytes(in[fpByteSize:]) + if err != nil { + return nil, err + } + z := new(fe).one() + p := &PointG1{*x, *y, *z} + if !g.IsOnCurve(p) { + return nil, errors.New("point is not on curve") + } + if !g.InCorrectSubgroup(p) { + return nil, errors.New("point is not on correct subgroup") + } + return p, nil +} + +// ToUncompressed given a G1 point returns bytes in uncompressed (x, y) form of the point. +// Serialization rules are in line with zcash library. See below for details. +// https://github.com/zcash/librustzcash/blob/master/pairing/src/bls12_381/README.md#serialization +// https://docs.rs/bls12_381/0.1.1/bls12_381/notes/serialization/index.html +func (g *G1) ToUncompressed(p *PointG1) []byte { + out := make([]byte, 2*fpByteSize) + if g.IsZero(p) { + out[0] |= 1 << 6 + return out + } + g.Affine(p) + copy(out[:fpByteSize], toBytes(&p[0])) + copy(out[fpByteSize:], toBytes(&p[1])) + return out +} + +// FromCompressed expects byte slice at least 48 bytes and given bytes returns a new point in G1. +// Serialization rules are in line with zcash library. See below for details. +// https://github.com/zcash/librustzcash/blob/master/pairing/src/bls12_381/README.md#serialization +// https://docs.rs/bls12_381/0.1.1/bls12_381/notes/serialization/index.html +func (g *G1) FromCompressed(compressed []byte) (*PointG1, error) { + if len(compressed) != fpByteSize { + return nil, errors.New("input string length must be equal to 48 bytes") + } + var in [fpByteSize]byte + copy(in[:], compressed[:]) + if in[0]&(1<<7) == 0 { + return nil, errors.New("compression flag must be set") + } + if in[0]&(1<<6) != 0 { + // in[0] == (1 << 6) + (1 << 7) + for i, v := range in { + if (i == 0 && v != 0xc0) || (i != 0 && v != 0x00) { + return nil, errors.New("input string must be zero when infinity flag is set") + } + } + return g.Zero(), nil + } + a := in[0]&(1<<5) != 0 + in[0] &= 0x1f + x, err := fromBytes(in[:]) + if err != nil { + return nil, err + } + // solve curve equation + y := &fe{} + square(y, x) + mul(y, y, x) + add(y, y, b) + if ok := sqrt(y, y); !ok { + return nil, errors.New("point is not on curve") + } + if y.signBE() == a { + neg(y, y) + } + z := new(fe).one() + p := &PointG1{*x, *y, *z} + if !g.InCorrectSubgroup(p) { + return nil, errors.New("point is not on correct subgroup") + } + return p, nil +} + +// ToCompressed given a G1 point returns bytes in compressed form of the point. +// Serialization rules are in line with zcash library. See below for details. +// https://github.com/zcash/librustzcash/blob/master/pairing/src/bls12_381/README.md#serialization +// https://docs.rs/bls12_381/0.1.1/bls12_381/notes/serialization/index.html +func (g *G1) ToCompressed(p *PointG1) []byte { + out := make([]byte, fpByteSize) + g.Affine(p) + if g.IsZero(p) { + out[0] |= 1 << 6 + } else { + copy(out[:], toBytes(&p[0])) + if !p[1].signBE() { + out[0] |= 1 << 5 + } + } + out[0] |= 1 << 7 + return out +} + +func (g *G1) fromBytesUnchecked(in []byte) (*PointG1, error) { + p0, err := fromBytes(in[:fpByteSize]) + if err != nil { + return nil, err + } + p1, err := fromBytes(in[fpByteSize:]) if err != nil { return nil, err } @@ -84,19 +199,17 @@ func (g *G1) fromBytesUnchecked(in []byte) (*PointG1, error) { } // FromBytes constructs a new point given uncompressed byte input. -// FromBytes does not take zcash flags into account. -// Byte input expected to be larger than 96 bytes. -// First 96 bytes should be concatenation of x and y values. -// Point (0, 0) is considered as infinity. +// Input string is expected to be equal to 96 bytes and concatenation of x and y cooridanates. +// (0, 0) is considered as infinity. func (g *G1) FromBytes(in []byte) (*PointG1, error) { - if len(in) != 96 { - return nil, errors.New("input string should be equal or larger than 96") + if len(in) != 2*fpByteSize { + return nil, errors.New("input string length must be equal to 96 bytes") } - p0, err := fromBytes(in[:48]) + p0, err := fromBytes(in[:fpByteSize]) if err != nil { return nil, err } - p1, err := fromBytes(in[48:]) + p1, err := fromBytes(in[fpByteSize:]) if err != nil { return nil, err } @@ -112,49 +225,16 @@ func (g *G1) FromBytes(in []byte) (*PointG1, error) { return p, nil } -// DecodePoint given encoded (x, y) coordinates in 128 bytes returns a valid G1 Point. -func (g *G1) DecodePoint(in []byte) (*PointG1, error) { - if len(in) != 128 { - return nil, errors.New("invalid g1 point length") - } - pointBytes := make([]byte, 96) - // decode x - xBytes, err := decodeFieldElement(in[:64]) - if err != nil { - return nil, err - } - // decode y - yBytes, err := decodeFieldElement(in[64:]) - if err != nil { - return nil, err - } - copy(pointBytes[:48], xBytes) - copy(pointBytes[48:], yBytes) - return g.FromBytes(pointBytes) -} - // ToBytes serializes a point into bytes in uncompressed form. -// ToBytes does not take zcash flags into account. // ToBytes returns (0, 0) if point is infinity. func (g *G1) ToBytes(p *PointG1) []byte { - out := make([]byte, 96) + out := make([]byte, 2*fpByteSize) if g.IsZero(p) { return out } g.Affine(p) - copy(out[:48], toBytes(&p[0])) - copy(out[48:], toBytes(&p[1])) - return out -} - -// EncodePoint encodes a point into 128 bytes. -func (g *G1) EncodePoint(p *PointG1) []byte { - outRaw := g.ToBytes(p) - out := make([]byte, 128) - // encode x - copy(out[16:], outRaw[:48]) - // encode y - copy(out[64+16:], outRaw[48:]) + copy(out[:fpByteSize], toBytes(&p[0])) + copy(out[fpByteSize:], toBytes(&p[1])) return out } @@ -201,9 +281,29 @@ func (g *G1) Equal(p1, p2 *PointG1) bool { // InCorrectSubgroup checks whether given point is in correct subgroup. func (g *G1) InCorrectSubgroup(p *PointG1) bool { - tmp := &PointG1{} - g.MulScalar(tmp, p, q) - return g.IsZero(tmp) + + // Faster Subgroup Checks for BLS12-381 + // S. Bowe + // https://eprint.iacr.org/2019/814.pdf + + mulZ := func(p *PointG1) { + // z = [(x^2 − 1)/3] + z := &Fr{0x0000000055555555, 0x396c8c005555e156} + e := z.toWNAF(wnafMulWindowG1) + g.wnafMul(p, p, e) + } + + // [(x^2 − 1)/3](2σ(P) − P − σ^2(P)) − σ^2(P) ?= O + t0 := g.New().Set(p) + g.glvEndomorphism(t0, t0) + t1 := g.New().Set(t0) // σ(P) + g.glvEndomorphism(t0, t0) // σ^2(P) + g.Double(t1, t1) // 2σ(P) + g.Sub(t1, t1, p) // 2σ(P) − P + g.Sub(t1, t1, t0) // 2σ(P) − P − σ^2(P) + mulZ(t1) // [(x^2 − 1)/3](2σ(P) − P − σ^2(P)) + g.Sub(t1, t1, t0) // [(x^2 − 1)/3](2σ(P) − P − σ^2(P)) − σ^2(P) + return g.IsZero(t1) } // IsOnCurve checks a G1 point is on curve. @@ -212,15 +312,19 @@ func (g *G1) IsOnCurve(p *PointG1) bool { return true } t := g.t - square(t[0], &p[1]) - square(t[1], &p[0]) - mul(t[1], t[1], &p[0]) - square(t[2], &p[2]) - square(t[3], t[2]) - mul(t[2], t[2], t[3]) - mul(t[2], b, t[2]) - add(t[1], t[1], t[2]) - return t[0].equal(t[1]) + square(t[0], &p[1]) // y^2 + square(t[1], &p[0]) // x^2 + mul(t[1], t[1], &p[0]) // x^3 + if p.IsAffine() { + addAssign(t[1], b) // x^2 + b + return t[0].equal(t[1]) // y^2 ?= x^3 + b + } + square(t[2], &p[2]) // z^2 + square(t[3], t[2]) // z^4 + mul(t[2], t[2], t[3]) // z^6 + mul(t[2], b, t[2]) // b * z^6 + add(t[1], t[1], t[2]) // x^3 + b * z^6 + return t[0].equal(t[1]) // y^2 ?= x^3 + b * z^6 } // IsAffine checks a G1 point whether it is in affine form. @@ -228,26 +332,105 @@ func (g *G1) IsAffine(p *PointG1) bool { return p[2].isOne() } -// Affine calculates affine form of given G1 point. +// Affine returns the affine representation of the given point func (g *G1) Affine(p *PointG1) *PointG1 { + return g.affine(p, p) +} + +func (g *G1) affine(r, p *PointG1) *PointG1 { if g.IsZero(p) { - return p + return r.Zero() } if !g.IsAffine(p) { t := g.t - inverse(t[0], &p[2]) - square(t[1], t[0]) - mul(&p[0], &p[0], t[1]) - mul(t[0], t[0], t[1]) - mul(&p[1], &p[1], t[0]) - p[2].one() + inverse(t[0], &p[2]) // z^-1 + square(t[1], t[0]) // z^-2 + mul(&r[0], &p[0], t[1]) // x = x * z^-2 + mul(t[0], t[0], t[1]) // z^-3 + mul(&r[1], &p[1], t[0]) // y = y * z^-3 + r[2].one() // z = 1 + } else { + r.Set(p) + } + return r +} + +// AffineBatch given multiple of points returns affine representations +func (g *G1) AffineBatch(p []*PointG1) { + inverses := make([]fe, len(p)) + for i := 0; i < len(p); i++ { + inverses[i].set(&p[i][2]) + } + inverseBatch(inverses) + t := g.t + for i := 0; i < len(p); i++ { + if !g.IsAffine(p[i]) && !g.IsZero(p[i]) { + square(t[1], &inverses[i]) + mul(&p[i][0], &p[i][0], t[1]) + mul(t[0], &inverses[i], t[1]) + mul(&p[i][1], &p[i][1], t[0]) + p[i][2].one() + } } - return p } // Add adds two G1 points p1, p2 and assigns the result to point at first argument. func (g *G1) Add(r, p1, p2 *PointG1) *PointG1 { - // www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl + + // http://www.hyperelliptic.org/EFD/gp/auto-shortw-jacobian-0.html#addition-add-2007-bl + if g.IsZero(p1) { + return r.Set(p2) + } + if g.IsZero(p2) { + return r.Set(p1) + } + if g.IsAffine(p2) { + return g.AddMixed(r, p1, p2) + } + t := g.t + square(t[7], &p1[2]) // z1z1 + mul(t[1], &p2[0], t[7]) // u2 = x2 * z1z1 + mul(t[2], &p1[2], t[7]) // z1z1 * z1 + mul(t[0], &p2[1], t[2]) // s2 = y2 * z1z1 * z1 + square(t[8], &p2[2]) // z2z2 + mul(t[3], &p1[0], t[8]) // u1 = x1 * z2z2 + mul(t[4], &p2[2], t[8]) // z2z2 * z2 + mul(t[2], &p1[1], t[4]) // s1 = y1 * z2z2 * z2 + if t[1].equal(t[3]) { + if t[0].equal(t[2]) { + return g.Double(r, p1) + } else { + return r.Zero() + } + } + subAssign(t[1], t[3]) // h = u2 - u1 + double(t[4], t[1]) // 2h + square(t[4], t[4]) // i = 2h^2 + mul(t[5], t[1], t[4]) // j = h*i + subAssign(t[0], t[2]) // s2 - s1 + doubleAssign(t[0]) // r = 2*(s2 - s1) + square(t[6], t[0]) // r^2 + subAssign(t[6], t[5]) // r^2 - j + mul(t[3], t[3], t[4]) // v = u1 * i + double(t[4], t[3]) // 2*v + sub(&r[0], t[6], t[4]) // x3 = r^2 - j - 2*v + sub(t[4], t[3], &r[0]) // v - x3 + mul(t[6], t[2], t[5]) // s1 * j + doubleAssign(t[6]) // 2 * s1 * j + mul(t[0], t[0], t[4]) // r * (v - x3) + sub(&r[1], t[0], t[6]) // y3 = r * (v - x3) - (2 * s1 * j) + add(t[0], &p1[2], &p2[2]) // z1 + z2 + square(t[0], t[0]) // (z1 + z2)^2 + subAssign(t[0], t[7]) // (z1 + z2)^2 - z1z1 + subAssign(t[0], t[8]) // (z1 + z2)^2 - z1z1 - z2z2 + mul(&r[2], t[0], t[1]) // z3 = ((z1 + z2)^2 - z1z1 - z2z2) * h + return r +} + +// Add adds two G1 points p1, p2 and assigns the result to point at first argument. +// Expects the second point p2 in affine form. +func (g *G1) AddMixed(r, p1, p2 *PointG1) *PointG1 { + // http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-madd-2007-bl if g.IsZero(p1) { return r.Set(p2) } @@ -255,73 +438,68 @@ func (g *G1) Add(r, p1, p2 *PointG1) *PointG1 { return r.Set(p1) } t := g.t - square(t[7], &p1[2]) - mul(t[1], &p2[0], t[7]) - mul(t[2], &p1[2], t[7]) - mul(t[0], &p2[1], t[2]) - square(t[8], &p2[2]) - mul(t[3], &p1[0], t[8]) - mul(t[4], &p2[2], t[8]) - mul(t[2], &p1[1], t[4]) - if t[1].equal(t[3]) { - if t[0].equal(t[2]) { - return g.Double(r, p1) - } - return r.Zero() + square(t[7], &p1[2]) // z1z1 + mul(t[1], &p2[0], t[7]) // u2 = x2 * z1z1 + mul(t[2], &p1[2], t[7]) // z1z1 * z1 + mul(t[0], &p2[1], t[2]) // s2 = y2 * z1z1 * z1 + + if p1[0].equal(t[1]) && p1[1].equal(t[0]) { + return g.Double(r, p1) } - sub(t[1], t[1], t[3]) - double(t[4], t[1]) - square(t[4], t[4]) - mul(t[5], t[1], t[4]) - sub(t[0], t[0], t[2]) - double(t[0], t[0]) - square(t[6], t[0]) - sub(t[6], t[6], t[5]) - mul(t[3], t[3], t[4]) - double(t[4], t[3]) - sub(&r[0], t[6], t[4]) - sub(t[4], t[3], &r[0]) - mul(t[6], t[2], t[5]) - double(t[6], t[6]) - mul(t[0], t[0], t[4]) - sub(&r[1], t[0], t[6]) - add(t[0], &p1[2], &p2[2]) - square(t[0], t[0]) - sub(t[0], t[0], t[7]) - sub(t[0], t[0], t[8]) - mul(&r[2], t[0], t[1]) + + sub(t[1], t[1], &p1[0]) // h = u2 - x1 + square(t[2], t[1]) // hh + double(t[4], t[2]) + doubleAssign(t[4]) // 4hh + mul(t[5], t[1], t[4]) // j = h*i + subAssign(t[0], &p1[1]) // s2 - y1 + doubleAssign(t[0]) // r = 2*(s2 - y1) + square(t[6], t[0]) // r^2 + subAssign(t[6], t[5]) // r^2 - j + mul(t[3], &p1[0], t[4]) // v = x1 * i + double(t[4], t[3]) // 2*v + sub(&r[0], t[6], t[4]) // x3 = r^2 - j - 2*v + sub(t[4], t[3], &r[0]) // v - x3 + mul(t[6], &p1[1], t[5]) // y1 * j + doubleAssign(t[6]) // 2 * y1 * j + mul(t[0], t[0], t[4]) // r * (v - x3) + sub(&r[1], t[0], t[6]) // y3 = r * (v - x3) - (2 * y1 * j) + add(t[0], &p1[2], t[1]) // z1 + h + square(t[0], t[0]) // (z1 + h)^2 + subAssign(t[0], t[7]) // (z1 + h)^2 - z1z1 + sub(&r[2], t[0], t[2]) // z3 = (z1 + z2)^2 - z1z1 - hh return r } // Double doubles a G1 point p and assigns the result to the point at first argument. func (g *G1) Double(r, p *PointG1) *PointG1 { - // http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#doubling-dbl-2009-l + // http://www.hyperelliptic.org/EFD/gp/auto-shortw-jacobian-0.html#doubling-dbl-2009-l if g.IsZero(p) { - return r.Set(p) + return r.Zero() } t := g.t - square(t[0], &p[0]) - square(t[1], &p[1]) - square(t[2], t[1]) - add(t[1], &p[0], t[1]) - square(t[1], t[1]) - sub(t[1], t[1], t[0]) - sub(t[1], t[1], t[2]) - double(t[1], t[1]) - double(t[3], t[0]) - add(t[0], t[3], t[0]) - square(t[4], t[0]) - double(t[3], t[1]) - sub(&r[0], t[4], t[3]) - sub(t[1], t[1], &r[0]) - double(t[2], t[2]) - double(t[2], t[2]) - double(t[2], t[2]) - mul(t[0], t[0], t[1]) - sub(t[1], t[0], t[2]) - mul(t[0], &p[1], &p[2]) - r[1].set(t[1]) - double(&r[2], t[0]) + square(t[0], &p[0]) // a = x^2 + square(t[1], &p[1]) // b = y^2 + square(t[2], t[1]) // c = b^2 + add(t[1], &p[0], t[1]) // b + x1 + square(t[1], t[1]) // (b + x1)^2 + subAssign(t[1], t[0]) // (b + x1)^2 - a + subAssign(t[1], t[2]) // (b + x1)^2 - a - c + doubleAssign(t[1]) // d = 2((b+x1)^2 - a - c) + double(t[3], t[0]) // 2a + addAssign(t[0], t[3]) // e = 3a + square(t[4], t[0]) // f = e^2 + double(t[3], t[1]) // 2d + sub(&r[0], t[4], t[3]) // x3 = f - 2d + subAssign(t[1], &r[0]) // d-x3 + doubleAssign(t[2]) // + doubleAssign(t[2]) // + doubleAssign(t[2]) // 8c + mul(t[0], t[0], t[1]) // e * (d - x3) + sub(t[1], t[0], t[2]) // x3 = e * (d - x3) - 8c + mul(t[0], &p[1], &p[2]) // y1 * z1 + r[1].set(t[1]) // + double(&r[2], t[0]) // z3 = 2(y1 * z1) return r } @@ -341,12 +519,32 @@ func (g *G1) Sub(c, a, b *PointG1) *PointG1 { return c } +// MulScalar multiplies a point by given scalar value and assigns the result to point at first argument. +func (g *G1) MulScalar(r, p *PointG1, e *Fr) *PointG1 { + return g.glvMulFr(r, p, e) +} + // MulScalar multiplies a point by given scalar value in big.Int and assigns the result to point at first argument. -func (g *G1) MulScalar(c, p *PointG1, e *big.Int) *PointG1 { +func (g *G1) MulScalarBig(r, p *PointG1, e *big.Int) *PointG1 { + return g.glvMulBig(r, p, e) +} + +func (g *G1) mulScalar(c, p *PointG1, e *Fr) *PointG1 { q, n := &PointG1{}, &PointG1{} n.Set(p) - l := e.BitLen() - for i := 0; i < l; i++ { + for i := 0; i < frBitSize; i++ { + if e.Bit(i) { + g.Add(q, q, n) + } + g.Double(n, n) + } + return c.Set(q) +} + +func (g *G1) mulScalarBig(c, p *PointG1, e *big.Int) *PointG1 { + q, n := &PointG1{}, &PointG1{} + n.Set(p) + for i := 0; i < frBitSize; i++ { if e.Bit(i) == 1 { g.Add(q, q, n) } @@ -355,67 +553,242 @@ func (g *G1) MulScalar(c, p *PointG1, e *big.Int) *PointG1 { return c.Set(q) } -// ClearCofactor maps given a G1 point to correct subgroup -func (g *G1) ClearCofactor(p *PointG1) { - g.MulScalar(p, p, cofactorEFFG1) +func (g *G1) wnafMulFr(r, p *PointG1, e *Fr) *PointG1 { + wnaf := e.toWNAF(wnafMulWindowG1) + return g.wnafMul(r, p, wnaf) } -// MultiExp calculates multi exponentiation. Given pairs of G1 point and scalar values -// (P_0, e_0), (P_1, e_1), ... (P_n, e_n) calculates r = e_0 * P_0 + e_1 * P_1 + ... + e_n * P_n -// Length of points and scalars are expected to be equal, otherwise an error is returned. -// Result is assigned to point at first argument. -func (g *G1) MultiExp(r *PointG1, points []*PointG1, powers []*big.Int) (*PointG1, error) { - if len(points) != len(powers) { - return nil, errors.New("point and scalar vectors should be in same length") +func (g *G1) wnafMulBig(r, p *PointG1, e *big.Int) *PointG1 { + wnaf := bigToWNAF(e, wnafMulWindowG1) + return g.wnafMul(r, p, wnaf) +} + +func (g *G1) wnafMul(c, p *PointG1, wnaf nafNumber) *PointG1 { + + l := (1 << (wnafMulWindowG1 - 1)) + + twoP, acc := g.New(), new(PointG1).Set(p) + g.Double(twoP, p) + g.Affine(twoP) + + // table = {p, 3p, 5p, ..., -p, -3p, -5p} + table := make([]*PointG1, l*2) + table[0], table[l] = g.New(), g.New() + table[0].Set(p) + g.Neg(table[l], table[0]) + + for i := 1; i < l; i++ { + g.AddMixed(acc, acc, twoP) + table[i], table[i+l] = g.New(), g.New() + table[i].Set(acc) + g.Neg(table[i+l], table[i]) } - var c uint32 = 3 - if len(powers) >= 32 { - c = uint32(math.Ceil(math.Log10(float64(len(powers))))) - } - bucketSize, numBits := (1<= 0; i-- { + if wnaf[i] > 0 { + g.Add(q, q, table[wnaf[i]>>1]) + } else if wnaf[i] < 0 { + g.Add(q, q, table[((-wnaf[i])>>1)+l]) } - for i := 0; i < len(powers); i++ { - s0 := powers[i].Uint64() - index := uint(s0 & mask) - if index != 0 { - g.Add(bucket[index-1], bucket[index-1], points[i]) + if i != 0 { + g.Double(q, q) + } + } + return c.Set(q) +} + +func (g *G1) glvMulFr(r, p *PointG1, e *Fr) *PointG1 { + return g.glvMul(r, p, new(glvVectorFr).new(e)) +} + +func (g *G1) glvMulBig(r, p *PointG1, e *big.Int) *PointG1 { + return g.glvMul(r, p, new(glvVectorBig).new(e)) +} + +func (g *G1) glvMul(r, p0 *PointG1, v glvVector) *PointG1 { + + w := glvMulWindowG1 + l := 1 << (w - 1) + + // prepare tables + // tableK1 = {P, 3P, 5P, ...} + // tableK2 = {λP, 3λP, 5λP, ...} + tableK1, tableK2 := make([]*PointG1, l), make([]*PointG1, l) + double := g.New() + g.Double(double, p0) + g.affine(double, double) + tableK1[0] = new(PointG1) + tableK1[0].Set(p0) + for i := 1; i < l; i++ { + tableK1[i] = new(PointG1) + g.AddMixed(tableK1[i], tableK1[i-1], double) + } + g.AffineBatch(tableK1) + for i := 0; i < l; i++ { + tableK2[i] = new(PointG1) + g.glvEndomorphism(tableK2[i], tableK1[i]) + } + + // recode small scalars + naf1, naf2 := v.wnaf(w) + lenNAF1, lenNAF2 := len(naf1), len(naf2) + lenNAF := lenNAF1 + if lenNAF2 > lenNAF { + lenNAF = lenNAF2 + } + + acc, p1 := g.New(), g.New() + + // function for naf addition + add := func(table []*PointG1, naf int) { + if naf != 0 { + nafAbs := naf + if nafAbs < 0 { + nafAbs = -nafAbs } - powers[i] = new(big.Int).Rsh(powers[i], uint(c)) + p1.Set(table[nafAbs>>1]) + if naf < 0 { + g.Neg(p1, p1) + } + g.AddMixed(acc, acc, p1) } - sum.Zero() - for i := len(bucket) - 1; i >= 0; i-- { - g.Add(sum, sum, bucket[i]) - g.Add(acc, acc, sum) - } - windows[j] = g.New() - windows[j].Set(acc) - j++ - cur += c } - acc.Zero() - for i := len(windows) - 1; i >= 0; i-- { - for j := uint32(0); j < c; j++ { + + // sliding + for i := lenNAF - 1; i >= 0; i-- { + if i < lenNAF1 { + add(tableK1, naf1[i]) + } + if i < lenNAF2 { + add(tableK2, naf2[i]) + } + if i != 0 { g.Double(acc, acc) } - g.Add(acc, acc, windows[i]) + } + return r.Set(acc) +} + +// MultiExpBig calculates multi exponentiation. Scalar values are received as big.Int type. +// Given pairs of G1 point and scalar values `(P_0, e_0), (P_1, e_1), ... (P_n, e_n)`, +// calculates `r = e_0 * P_0 + e_1 * P_1 + ... + e_n * P_n`. +// Length of points and scalars are expected to be equal, otherwise an error is returned. +// Result is assigned to point at first argument. +func (g *G1) MultiExpBig(r *PointG1, points []*PointG1, scalars []*big.Int) (*PointG1, error) { + if len(points) != len(scalars) { + return nil, errors.New("point and scalar vectors should be in same length") + } + + c := 3 + if len(scalars) >= 32 { + c = int(math.Ceil(math.Log(float64(len(scalars))))) + } + + bucketSize := (1 << c) - 1 + windows := make([]PointG1, 255/c+1) + bucket := make([]PointG1, bucketSize) + + for j := 0; j < len(windows); j++ { + + for i := 0; i < bucketSize; i++ { + bucket[i].Zero() + } + + for i := 0; i < len(scalars); i++ { + index := bucketSize & int(new(big.Int).Rsh(scalars[i], uint(c*j)).Int64()) + if index != 0 { + g.Add(&bucket[index-1], &bucket[index-1], points[i]) + } + } + + acc, sum := g.New(), g.New() + for i := bucketSize - 1; i >= 0; i-- { + g.Add(sum, sum, &bucket[i]) + g.Add(acc, acc, sum) + } + windows[j].Set(acc) + } + + acc := g.New() + for i := len(windows) - 1; i >= 0; i-- { + for j := 0; j < c; j++ { + g.Double(acc, acc) + } + g.Add(acc, acc, &windows[i]) } return r.Set(acc), nil } +// MultiExp calculates multi exponentiation. Given pairs of G1 point and scalar values `(P_0, e_0), (P_1, e_1), ... (P_n, e_n)`, +// calculates `r = e_0 * P_0 + e_1 * P_1 + ... + e_n * P_n`. Length of points and scalars are expected to be equal, +// otherwise an error is returned. Result is assigned to point at first argument. +func (g *G1) MultiExp(r *PointG1, points []*PointG1, scalars []*Fr) (*PointG1, error) { + if len(points) != len(scalars) { + return nil, errors.New("point and scalar vectors should be in same length") + } + + g.AffineBatch(points) + + c := 3 + if len(scalars) >= 32 { + c = int(math.Ceil(math.Log(float64(len(scalars))))) + } + + bucketSize := (1 << c) - 1 + windows := make([]*PointG1, 255/c+1) + bucket := make([]PointG1, bucketSize) + + for j := 0; j < len(windows); j++ { + + for i := 0; i < bucketSize; i++ { + bucket[i].Zero() + } + + for i := 0; i < len(scalars); i++ { + index := bucketSize & int(scalars[i].sliceUint64(c*j)) + if index != 0 { + g.AddMixed(&bucket[index-1], &bucket[index-1], points[i]) + } + } + + acc, sum := g.New(), g.New() + for i := bucketSize - 1; i >= 0; i-- { + g.Add(sum, sum, &bucket[i]) + g.Add(acc, acc, sum) + } + windows[j] = g.New().Set(acc) + } + + g.AffineBatch(windows) + + acc := g.New() + for i := len(windows) - 1; i >= 0; i-- { + for j := 0; j < c; j++ { + g.Double(acc, acc) + } + g.AddMixed(acc, acc, windows[i]) + } + return r.Set(acc), nil +} + +func (g *G1) ClearCofactor(p *PointG1) *PointG1 { + chain := func(p0 *PointG1, n int, p1 *PointG1) { + for i := 0; i < n; i++ { + g.Double(p0, p0) + } + g.Add(p0, p0, p1) + } + t := g.New().Set(p) + chain(p, 1, t) + chain(p, 2, t) + chain(p, 3, t) + chain(p, 9, t) + chain(p, 32, t) + chain(p, 16, t) + return p +} + // MapToCurve given a byte slice returns a valid G1 point. // This mapping function implements the Simplified Shallue-van de Woestijne-Ulas method. // https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06 @@ -432,3 +805,42 @@ func (g *G1) MapToCurve(in []byte) (*PointG1, error) { g.ClearCofactor(p) return g.Affine(p), nil } + +// EncodeToCurve given a message and domain seperator tag returns the hash result +// which is a valid curve point. +// Implementation follows BLS12381G1_XMD:SHA-256_SSWU_NU_ suite at +// https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06 +func (g *G1) EncodeToCurve(msg, domain []byte) (*PointG1, error) { + hashRes, err := hashToFpXMDSHA256(msg, domain, 1) + if err != nil { + return nil, err + } + u := hashRes[0] + x, y := swuMapG1(u) + isogenyMapG1(x, y) + one := new(fe).one() + p := &PointG1{*x, *y, *one} + g.ClearCofactor(p) + return g.Affine(p), nil +} + +// HashToCurve given a message and domain seperator tag returns the hash result +// which is a valid curve point. +// Implementation follows BLS12381G1_XMD:SHA-256_SSWU_RO_ suite at +// https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06 +func (g *G1) HashToCurve(msg, domain []byte) (*PointG1, error) { + hashRes, err := hashToFpXMDSHA256(msg, domain, 2) + if err != nil { + return nil, err + } + u0, u1 := hashRes[0], hashRes[1] + x0, y0 := swuMapG1(u0) + x1, y1 := swuMapG1(u1) + one := new(fe).one() + p0, p1 := &PointG1{*x0, *y0, *one}, &PointG1{*x1, *y1, *one} + g.Add(p0, p0, p1) + g.Affine(p0) + isogenyMapG1(&p0[0], &p0[1]) + g.ClearCofactor(p0) + return g.Affine(p0), nil +} diff --git a/crypto/bls12381/g1_test.go b/crypto/bls12381/g1_test.go index 87140459fb..690de846df 100644 --- a/crypto/bls12381/g1_test.go +++ b/crypto/bls12381/g1_test.go @@ -3,52 +3,117 @@ package bls12381 import ( "bytes" "crypto/rand" + "fmt" + "io/ioutil" "math/big" "testing" - - "github.com/ethereum/go-ethereum/common" ) func (g *G1) one() *PointG1 { - one, _ := g.fromBytesUnchecked( - common.FromHex("" + - "17f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb" + - "08b3f481e3aaa0f1a09e30ed741d8ae4fcf5e095d5d00af600db18cb2c04b3edd03cc744a2888ae40caa232946c5e7e1", - ), - ) - return one + return g.New().Set(&g1One) } func (g *G1) rand() *PointG1 { - k, err := rand.Int(rand.Reader, q) - if err != nil { - panic(err) + p := &PointG1{} + z, _ := new(fe).rand(rand.Reader) + z6, bz6 := new(fe), new(fe) + square(z6, z) + square(z6, z6) + mul(z6, z6, z) + mul(z6, z6, z) + mul(bz6, z6, b) + for { + x, _ := new(fe).rand(rand.Reader) + y := new(fe) + square(y, x) + mul(y, y, x) + add(y, y, bz6) + if sqrt(y, y) { + p.Set(&PointG1{*x, *y, *z}) + break + } } - return g.MulScalar(&PointG1{}, g.one(), k) + if !g.IsOnCurve(p) { + panic("rand point must be on curve") + } + if g.InCorrectSubgroup(p) { + panic("rand point must be out of correct subgroup") + } + return p +} + +func (g *G1) randCorrect() *PointG1 { + p := g.ClearCofactor(g.rand()) + if !g.InCorrectSubgroup(p) { + panic("must be in correct subgroup") + } + return p +} + +func (g *G1) randAffine() *PointG1 { + return g.Affine(g.randCorrect()) +} + +func (g *G1) new() *PointG1 { + return g.Zero() } func TestG1Serialization(t *testing.T) { - g1 := NewG1() + var err error + g := NewG1() + zero := g.Zero() + b0 := g.ToUncompressed(zero) + p0, err := g.FromUncompressed(b0) + if err != nil { + t.Fatal(err) + } + if !g.IsZero(p0) { + t.Fatal("infinity serialization failed") + } + b0 = g.ToCompressed(zero) + p0, err = g.FromCompressed(b0) + if err != nil { + t.Fatal(err) + } + if !g.IsZero(p0) { + t.Fatal("infinity serialization failed") + } + b0 = g.ToBytes(zero) + p0, err = g.FromBytes(b0) + if err != nil { + t.Fatal(err) + } + if !g.IsZero(p0) { + t.Fatal("infinity serialization failed") + } for i := 0; i < fuz; i++ { - a := g1.rand() - buf := g1.ToBytes(a) - b, err := g1.FromBytes(buf) + a := g.randAffine() + uncompressed := g.ToUncompressed(a) + b, err := g.FromUncompressed(uncompressed) if err != nil { t.Fatal(err) } - if !g1.Equal(a, b) { - t.Fatal("bad serialization from/to") + if !g.Equal(a, b) { + t.Fatal("serialization failed") + } + compressed := g.ToCompressed(b) + a, err = g.FromCompressed(compressed) + if err != nil { + t.Fatal(err) + } + if !g.Equal(a, b) { + t.Fatal("serialization failed") } } for i := 0; i < fuz; i++ { - a := g1.rand() - encoded := g1.EncodePoint(a) - b, err := g1.DecodePoint(encoded) + a := g.randAffine() + uncompressed := g.ToBytes(a) + b, err := g.FromBytes(uncompressed) if err != nil { t.Fatal(err) } - if !g1.Equal(a, b) { - t.Fatal("bad serialization encode/decode") + if !g.Equal(a, b) { + t.Fatal("serialization failed") } } } @@ -66,6 +131,26 @@ func TestG1IsOnCurve(t *testing.T) { } } +func TestG1BatchAffine(t *testing.T) { + n := 20 + g := NewG1() + points0 := make([]*PointG1, n) + points1 := make([]*PointG1, n) + for i := 0; i < n; i++ { + points0[i] = g.rand() + points1[i] = g.New().Set(points0[i]) + if g.IsAffine(points0[i]) { + t.Fatal("expect non affine point") + } + } + g.AffineBatch(points0) + for i := 0; i < n; i++ { + if !g.Equal(points0[i], points1[i]) { + t.Fatal("batch affine failed") + } + } +} + func TestG1AdditiveProperties(t *testing.T) { g := NewG1() t0, t1 := g.New(), g.New() @@ -135,14 +220,71 @@ func TestG1AdditiveProperties(t *testing.T) { } } +func TestG1MixedAdd(t *testing.T) { + g := NewG1() + for i := 0; i < fuz; i++ { + a, b := g.rand(), g.rand() + if g.IsAffine(a) || g.IsAffine(b) { + t.Fatal("expect non affine points") + } + bAffine := g.New().Set(b) + g.Affine(bAffine) + r0, r1 := g.New(), g.New() + g.Add(r0, a, b) + g.AddMixed(r1, a, bAffine) + if !g.Equal(r0, r1) { + t.Fatal("mixed addition failed") + } + aAffine := g.New().Set(a) + g.Affine(aAffine) + g.AddMixed(r0, a, aAffine) + g.Double(r1, a) + if !g.Equal(r0, r1) { + t.Fatal("mixed addition must double where points are equal") + } + } +} + +func TestG1MultiplicationCross(t *testing.T) { + g := NewG1() + for i := 0; i < fuz; i++ { + + a := g.randCorrect() + s, _ := new(Fr).Rand(rand.Reader) + sBig := s.ToBig() + res0, res1, res2, res3, res4 := g.New(), g.New(), g.New(), g.New(), g.New() + + g.mulScalar(res0, a, s) + g.glvMulFr(res1, a, s) + g.glvMulBig(res2, a, sBig) + g.wnafMulFr(res3, a, s) + g.wnafMulBig(res4, a, sBig) + + if !g.Equal(res0, res1) { + t.Fatal("cross multiplication failed (glv, fr)", i) + } + if !g.Equal(res0, res2) { + t.Fatal("cross multiplication failed (glv, big)", i) + } + if !g.Equal(res0, res3) { + t.Fatal("cross multiplication failed (wnaf, fr)", i) + } + if !g.Equal(res0, res4) { + t.Fatal("cross multiplication failed (wnaf, big)", i) + } + } +} + func TestG1MultiplicativeProperties(t *testing.T) { g := NewG1() t0, t1 := g.New(), g.New() zero := g.Zero() for i := 0; i < fuz; i++ { - a := g.rand() - s1, s2, s3 := randScalar(q), randScalar(q), randScalar(q) - sone := big.NewInt(1) + a := g.randCorrect() + s1, _ := new(Fr).Rand(rand.Reader) + s2, _ := new(Fr).Rand(rand.Reader) + s3, _ := new(Fr).Rand(rand.Reader) + sone := &Fr{1} g.MulScalar(t0, zero, s1) if !g.Equal(t0, zero) { t.Fatal(" 0 ^ s == 0") @@ -160,7 +302,7 @@ func TestG1MultiplicativeProperties(t *testing.T) { s3.Mul(s1, s2) g.MulScalar(t1, a, s3) if !g.Equal(t0, t1) { - t.Errorf(" (a ^ s1) ^ s2 == a ^ (s1 * s2)") + t.Fatal(" (a ^ s1) ^ s2 == a ^ (s1 * s2)") } g.MulScalar(t0, a, s1) g.MulScalar(t1, a, s2) @@ -168,12 +310,71 @@ func TestG1MultiplicativeProperties(t *testing.T) { s3.Add(s1, s2) g.MulScalar(t1, a, s3) if !g.Equal(t0, t1) { - t.Errorf(" (a ^ s1) + (a ^ s2) == a ^ (s1 + s2)") + t.Fatal(" (a ^ s1) + (a ^ s2) == a ^ (s1 + s2)") } } } +func TestZKCryptoVectorsG1UncompressedValid(t *testing.T) { + data, err := ioutil.ReadFile("tests/g1_uncompressed_valid_test_vectors.dat") + if err != nil { + panic(err) + } + g := NewG1() + p1 := g.Zero() + for i := 0; i < 1000; i++ { + vector := data[i*2*fpByteSize : (i+1)*2*fpByteSize] + p2, err := g.FromUncompressed(vector) + if err != nil { + t.Fatal("decoing fails", err, i) + } + uncompressed := g.ToUncompressed(p2) + if !bytes.Equal(vector, uncompressed) || !g.Equal(p1, p2) { + t.Fatal("serialization failed") + } + + g.Add(p1, p1, &g1One) + } +} + +func TestZKCryptoVectorsG1CompressedValid(t *testing.T) { + data, err := ioutil.ReadFile("tests/g1_compressed_valid_test_vectors.dat") + if err != nil { + panic(err) + } + g := NewG1() + p1 := g.Zero() + for i := 0; i < 1000; i++ { + vector := data[i*fpByteSize : (i+1)*fpByteSize] + p2, err := g.FromCompressed(vector) + if err != nil { + t.Fatal("decoing fails", err, i) + } + compressed := g.ToCompressed(p2) + if !bytes.Equal(vector, compressed) || !g.Equal(p1, p2) { + t.Fatal("serialization failed") + } + g.Add(p1, p1, &g1One) + } +} + func TestG1MultiExpExpected(t *testing.T) { + g := NewG1() + one := g.one() + var scalars [2]*Fr + var bases [2]*PointG1 + scalars[0] = &Fr{2} + scalars[1] = &Fr{3} + bases[0], bases[1] = new(PointG1).Set(one), new(PointG1).Set(one) + expected, result := g.New(), g.New() + g.mulScalar(expected, one, &Fr{5}) + _, _ = g.MultiExp(result, bases[:], scalars[:]) + if !g.Equal(expected, result) { + t.Fatal("multi-exponentiation failed") + } +} + +func TestG1MultiExpBigExpected(t *testing.T) { g := NewG1() one := g.one() var scalars [2]*big.Int @@ -182,36 +383,76 @@ func TestG1MultiExpExpected(t *testing.T) { scalars[1] = big.NewInt(3) bases[0], bases[1] = new(PointG1).Set(one), new(PointG1).Set(one) expected, result := g.New(), g.New() - g.MulScalar(expected, one, big.NewInt(5)) - _, _ = g.MultiExp(result, bases[:], scalars[:]) + g.mulScalarBig(expected, one, big.NewInt(5)) + _, _ = g.MultiExpBig(result, bases[:], scalars[:]) if !g.Equal(expected, result) { - t.Fatal("bad multi-exponentiation") + t.Fatal("multi-exponentiation failed") } } -func TestG1MultiExpBatch(t *testing.T) { +func TestG1MultiExpBig(t *testing.T) { g := NewG1() - one := g.one() - n := 1000 - bases := make([]*PointG1, n) - scalars := make([]*big.Int, n) - // scalars: [s0,s1 ... s(n-1)] - // bases: [P0,P1,..P(n-1)] = [s(n-1)*G, s(n-2)*G ... s0*G] - for i, j := 0, n-1; i < n; i, j = i+1, j-1 { - scalars[j], _ = rand.Int(rand.Reader, big.NewInt(100000)) - bases[i] = g.New() - g.MulScalar(bases[i], one, scalars[j]) + for n := 1; n < 1024+1; n = n * 2 { + bases := make([]*PointG1, n) + scalars := make([]*big.Int, n) + var err error + for i := 0; i < n; i++ { + scalars[i], err = rand.Int(rand.Reader, qBig) + if err != nil { + t.Fatal(err) + } + bases[i] = g.randAffine() + } + expected, tmp := g.New(), g.New() + for i := 0; i < n; i++ { + g.mulScalarBig(tmp, bases[i], scalars[i]) + g.Add(expected, expected, tmp) + } + result := g.New() + _, _ = g.MultiExpBig(result, bases, scalars) + if !g.Equal(expected, result) { + t.Fatal("multi-exponentiation failed") + } } - // expected: s(n-1)*P0 + s(n-2)*P1 + s0*P(n-1) - expected, tmp := g.New(), g.New() - for i := 0; i < n; i++ { - g.MulScalar(tmp, bases[i], scalars[i]) - g.Add(expected, expected, tmp) +} + +func TestG1MultiExp(t *testing.T) { + g := NewG1() + for n := 1; n < 1024+1; n = n * 2 { + bases := make([]*PointG1, n) + scalars := make([]*Fr, n) + var err error + for i := 0; i < n; i++ { + scalars[i], err = new(Fr).Rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + bases[i] = g.randAffine() + } + expected, tmp := g.New(), g.New() + for i := 0; i < n; i++ { + g.mulScalar(tmp, bases[i], scalars[i]) + g.Add(expected, expected, tmp) + } + result := g.New() + _, _ = g.MultiExp(result, bases, scalars) + if !g.Equal(expected, result) { + t.Fatal("multi-exponentiation failed") + } } - result := g.New() - _, _ = g.MultiExp(result, bases, scalars) - if !g.Equal(expected, result) { - t.Fatal("bad multi-exponentiation") +} + +func TestG1ClearCofactor(t *testing.T) { + g := NewG1() + for i := 0; i < fuz; i++ { + p0 := g.rand() + if g.InCorrectSubgroup(p0) { + t.Fatal("rand point should be out of correct subgroup") + } + g.ClearCofactor(p0) + if !g.InCorrectSubgroup(p0) { + t.Fatal("cofactor clearing is failed") + } } } @@ -221,24 +462,39 @@ func TestG1MapToCurve(t *testing.T) { expected []byte }{ { - u: make([]byte, 48), - expected: common.FromHex("11a9a0372b8f332d5c30de9ad14e50372a73fa4c45d5f2fa5097f2d6fb93bcac592f2e1711ac43db0519870c7d0ea415" + "092c0f994164a0719f51c24ba3788de240ff926b55f58c445116e8bc6a47cd63392fd4e8e22bdf9feaa96ee773222133"), + u: make([]byte, fpByteSize), + expected: fromHex(-1, + "11a9a0372b8f332d5c30de9ad14e50372a73fa4c45d5f2fa5097f2d6fb93bcac592f2e1711ac43db0519870c7d0ea415", + "092c0f994164a0719f51c24ba3788de240ff926b55f58c445116e8bc6a47cd63392fd4e8e22bdf9feaa96ee773222133", + ), }, { - u: common.FromHex("07fdf49ea58e96015d61f6b5c9d1c8f277146a533ae7fbca2a8ef4c41055cd961fbc6e26979b5554e4b4f22330c0e16d"), - expected: common.FromHex("1223effdbb2d38152495a864d78eee14cb0992d89a241707abb03819a91a6d2fd65854ab9a69e9aacb0cbebfd490732c" + "0f925d61e0b235ecd945cbf0309291878df0d06e5d80d6b84aa4ff3e00633b26f9a7cb3523ef737d90e6d71e8b98b2d5"), + u: fromHex(-1, "07fdf49ea58e96015d61f6b5c9d1c8f277146a533ae7fbca2a8ef4c41055cd961fbc6e26979b5554e4b4f22330c0e16d"), + expected: fromHex(-1, + "1223effdbb2d38152495a864d78eee14cb0992d89a241707abb03819a91a6d2fd65854ab9a69e9aacb0cbebfd490732c", + "0f925d61e0b235ecd945cbf0309291878df0d06e5d80d6b84aa4ff3e00633b26f9a7cb3523ef737d90e6d71e8b98b2d5", + ), }, { - u: common.FromHex("1275ab3adbf824a169ed4b1fd669b49cf406d822f7fe90d6b2f8c601b5348436f89761bb1ad89a6fb1137cd91810e5d2"), - expected: common.FromHex("179d3fd0b4fb1da43aad06cea1fb3f828806ddb1b1fa9424b1e3944dfdbab6e763c42636404017da03099af0dcca0fd6" + "0d037cb1c6d495c0f5f22b061d23f1be3d7fe64d3c6820cfcd99b6b36fa69f7b4c1f4addba2ae7aa46fb25901ab483e4"), + u: fromHex(-1, "1275ab3adbf824a169ed4b1fd669b49cf406d822f7fe90d6b2f8c601b5348436f89761bb1ad89a6fb1137cd91810e5d2"), + expected: fromHex(-1, + "179d3fd0b4fb1da43aad06cea1fb3f828806ddb1b1fa9424b1e3944dfdbab6e763c42636404017da03099af0dcca0fd6", + "0d037cb1c6d495c0f5f22b061d23f1be3d7fe64d3c6820cfcd99b6b36fa69f7b4c1f4addba2ae7aa46fb25901ab483e4", + ), }, { - u: common.FromHex("0e93d11d30de6d84b8578827856f5c05feef36083eef0b7b263e35ecb9b56e86299614a042e57d467fa20948e8564909"), - expected: common.FromHex("15aa66c77eded1209db694e8b1ba49daf8b686733afaa7b68c683d0b01788dfb0617a2e2d04c0856db4981921d3004af" + "0952bb2f61739dd1d201dd0a79d74cda3285403d47655ee886afe860593a8a4e51c5b77a22d2133e3a4280eaaaa8b788"), + u: fromHex(-1, "0e93d11d30de6d84b8578827856f5c05feef36083eef0b7b263e35ecb9b56e86299614a042e57d467fa20948e8564909"), + expected: fromHex(-1, + "15aa66c77eded1209db694e8b1ba49daf8b686733afaa7b68c683d0b01788dfb0617a2e2d04c0856db4981921d3004af", + "0952bb2f61739dd1d201dd0a79d74cda3285403d47655ee886afe860593a8a4e51c5b77a22d2133e3a4280eaaaa8b788", + ), }, { - u: common.FromHex("015a41481155d17074d20be6d8ec4d46632a51521cd9c916e265bd9b47343b3689979b50708c8546cbc2916b86cb1a3a"), - expected: common.FromHex("06328ce5106e837935e8da84bd9af473422e62492930aa5f460369baad9545defa468d9399854c23a75495d2a80487ee" + "094bfdfe3e552447433b5a00967498a3f1314b86ce7a7164c8a8f4131f99333b30a574607e301d5f774172c627fd0bca"), + u: fromHex(-1, "015a41481155d17074d20be6d8ec4d46632a51521cd9c916e265bd9b47343b3689979b50708c8546cbc2916b86cb1a3a"), + expected: fromHex(-1, + "06328ce5106e837935e8da84bd9af473422e62492930aa5f460369baad9545defa468d9399854c23a75495d2a80487ee", + "094bfdfe3e552447433b5a00967498a3f1314b86ce7a7164c8a8f4131f99333b30a574607e301d5f774172c627fd0bca", + ), }, } { g := NewG1() @@ -252,31 +508,217 @@ func TestG1MapToCurve(t *testing.T) { } } -func BenchmarkG1Add(t *testing.B) { - g1 := NewG1() - a, b, c := g1.rand(), g1.rand(), PointG1{} - t.ResetTimer() - for i := 0; i < t.N; i++ { - g1.Add(&c, a, b) +func TestG1EncodeToCurve(t *testing.T) { + domain := []byte("BLS12381G1_XMD:SHA-256_SSWU_NU_TESTGEN") + for i, v := range []struct { + msg []byte + expected []byte + }{ + { + msg: []byte(""), + expected: fromHex(-1, + "1223effdbb2d38152495a864d78eee14cb0992d89a241707abb03819a91a6d2fd65854ab9a69e9aacb0cbebfd490732c", + "0f925d61e0b235ecd945cbf0309291878df0d06e5d80d6b84aa4ff3e00633b26f9a7cb3523ef737d90e6d71e8b98b2d5", + ), + }, + { + msg: []byte("abc"), + expected: fromHex(-1, + "179d3fd0b4fb1da43aad06cea1fb3f828806ddb1b1fa9424b1e3944dfdbab6e763c42636404017da03099af0dcca0fd6", + "0d037cb1c6d495c0f5f22b061d23f1be3d7fe64d3c6820cfcd99b6b36fa69f7b4c1f4addba2ae7aa46fb25901ab483e4", + ), + }, + { + msg: []byte("abcdef0123456789"), + expected: fromHex(-1, + "15aa66c77eded1209db694e8b1ba49daf8b686733afaa7b68c683d0b01788dfb0617a2e2d04c0856db4981921d3004af", + "0952bb2f61739dd1d201dd0a79d74cda3285403d47655ee886afe860593a8a4e51c5b77a22d2133e3a4280eaaaa8b788", + ), + }, + { + msg: []byte("a512_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + expected: fromHex(-1, + "06328ce5106e837935e8da84bd9af473422e62492930aa5f460369baad9545defa468d9399854c23a75495d2a80487ee", + "094bfdfe3e552447433b5a00967498a3f1314b86ce7a7164c8a8f4131f99333b30a574607e301d5f774172c627fd0bca", + ), + }, + } { + g := NewG1() + p0, err := g.EncodeToCurve(v.msg, domain) + if err != nil { + t.Fatal("encode to point fails", i, err) + } + if !bytes.Equal(g.ToBytes(p0), v.expected) { + t.Fatal("encode to point fails", i) + } } } -func BenchmarkG1Mul(t *testing.B) { - worstCaseScalar, _ := new(big.Int).SetString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16) - g1 := NewG1() - a, e, c := g1.rand(), worstCaseScalar, PointG1{} +func TestG1HashToCurve(t *testing.T) { + domain := []byte("BLS12381G1_XMD:SHA-256_SSWU_RO_TESTGEN") + for i, v := range []struct { + msg []byte + expected []byte + }{ + { + msg: []byte(""), + expected: fromHex(-1, + "0576730ab036cbac1d95b38dca905586f28d0a59048db4e8778782d89bff856ddef89277ead5a21e2975c4a6e3d8c79e", + "1273e568bebf1864393c517f999b87c1eaa1b8432f95aea8160cd981b5b05d8cd4a7cf00103b6ef87f728e4b547dd7ae", + ), + }, + { + msg: []byte("abc"), + expected: fromHex(-1, + "061daf0cc00d8912dac1d4cf5a7c32fca97f8b3bf3f805121888e5eb89f77f9a9f406569027ac6d0e61b1229f42c43d6", + "0de1601e5ba02cb637c1d35266f5700acee9850796dc88e860d022d7b9e7e3dce5950952e97861e5bb16d215c87f030d", + ), + }, + { + msg: []byte("abcdef0123456789"), + expected: fromHex(-1, + "0fb3455436843e76079c7cf3dfef75e5a104dfe257a29a850c145568d500ad31ccfe79be9ae0ea31a722548070cf98cd", + "177989f7e2c751658df1b26943ee829d3ebcf131d8f805571712f3a7527ee5334ecff8a97fc2a50cea86f5e6212e9a57", + ), + }, + { + msg: []byte("a512_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + expected: fromHex(-1, + "0514af2137c1ae1d78d5cb97ee606ea142824c199f0f25ac463a0c78200de57640d34686521d3e9cf6b3721834f8a038", + "047a85d6898416a0899e26219bca7c4f0fa682717199de196b02b95eaf9fb55456ac3b810e78571a1b7f5692b7c58ab6", + ), + }, + } { + g := NewG1() + p0, err := g.HashToCurve(v.msg, domain) + if err != nil { + t.Fatal("hash to point fails", i, err) + } + if !bytes.Equal(g.ToBytes(p0), v.expected) { + t.Fatal("hash to point fails", i) + } + } +} + +func BenchmarkG1Add(t *testing.B) { + g := NewG1() + a, b, c := g.rand(), g.rand(), PointG1{} t.ResetTimer() for i := 0; i < t.N; i++ { - g1.MulScalar(&c, a, e) + g.Add(&c, a, b) + } +} + +func BenchmarkG1MulWNAF(t *testing.B) { + g := NewG1() + p := new(PointG1).Set(&g1One) + s, _ := new(Fr).Rand(rand.Reader) + sBig := s.ToBig() + res := new(PointG1) + t.Run("Naive", func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.mulScalar(res, p, s) + } + }) + for i := 1; i < 8; i++ { + wnafMulWindowG1 = uint(i) + t.Run(fmt.Sprintf("Fr, window: %d", i), func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.wnafMulFr(res, p, s) + } + }) + t.Run(fmt.Sprintf("Big, window: %d", i), func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.wnafMulBig(res, p, sBig) + } + }) + } +} + +func BenchmarkG1MulGLV(t *testing.B) { + + g := NewG1() + p := new(PointG1).Set(&g1One) + s, _ := new(Fr).Rand(rand.Reader) + sBig := s.ToBig() + res := new(PointG1) + t.Run("Naive", func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.mulScalar(res, p, s) + } + }) + for i := 1; i < 8; i++ { + glvMulWindowG1 = uint(i) + t.Run(fmt.Sprintf("Fr, window: %d", i), func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.glvMulFr(res, p, s) + } + }) + t.Run(fmt.Sprintf("Big, window: %d", i), func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.glvMulBig(res, p, sBig) + } + }) + } +} + +func BenchmarkG1MultiExp(t *testing.B) { + g := NewG1() + v := func(n int) ([]*PointG1, []*Fr) { + bases := make([]*PointG1, n) + scalars := make([]*Fr, n) + var err error + for i := 0; i < n; i++ { + scalars[i], err = new(Fr).Rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + bases[i] = g.randAffine() + } + return bases, scalars + } + for _, i := range []int{2, 10, 100, 1000} { + t.Run(fmt.Sprint(i), func(t *testing.B) { + bases, scalars := v(i) + result := g.New() + t.ResetTimer() + for i := 0; i < t.N; i++ { + _, _ = g.MultiExp(result, bases, scalars) + } + }) + } +} + +func BenchmarkG1ClearCofactor(t *testing.B) { + g := NewG1() + a := g.rand() + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.ClearCofactor(a) + } +} + +func BenchmarkG1SubgroupCheck(t *testing.B) { + g := NewG1() + a := g.rand() + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.InCorrectSubgroup(a) } } func BenchmarkG1MapToCurve(t *testing.B) { - a := make([]byte, 48) - g1 := NewG1() + a := fromHex(fpByteSize, "0x1234") + g := NewG1() t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := g1.MapToCurve(a) + _, err := g.MapToCurve(a) if err != nil { t.Fatal(err) } diff --git a/crypto/bls12381/g2.go b/crypto/bls12381/g2.go index b942bf94fd..370d0f4339 100644 --- a/crypto/bls12381/g2.go +++ b/crypto/bls12381/g2.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( @@ -22,9 +6,8 @@ import ( "math/big" ) -// PointG2 is type for point in G2. -// PointG2 is both used for Affine and Jacobian point representation. -// If z is equal to one the point is considered as in affine form. +// PointG2 is type for point in G2 and used for both affine and Jacobian representation. +// A point is accounted as in affine form if z is equal to one. type PointG2 [3]fe2 // Set copies values of one point to another. @@ -35,7 +18,6 @@ func (p *PointG2) Set(p2 *PointG2) *PointG2 { return p } -// Zero returns G2 point in point at infinity representation func (p *PointG2) Zero() *PointG2 { p[0].zero() p[1].one() @@ -43,6 +25,11 @@ func (p *PointG2) Zero() *PointG2 { return p } +// IsAffine checks a G1 point whether it is in affine form. +func (p *PointG2) IsAffine() bool { + return p[2].isOne() +} + type tempG2 struct { t [9]*fe2 } @@ -76,15 +63,141 @@ func newTempG2() tempG2 { // Q returns group order in big.Int. func (g *G2) Q() *big.Int { - return new(big.Int).Set(q) + return new(big.Int).Set(qBig) } -func (g *G2) fromBytesUnchecked(in []byte) (*PointG2, error) { - p0, err := g.f.fromBytes(in[:96]) +// FromUncompressed expects byte slice at least 192 bytes and given bytes returns a new point in G2. +// Serialization rules are in line with zcash library. See below for details. +// https://github.com/zcash/librustzcash/blob/master/pairing/src/bls12_381/README.md#serialization +// https://docs.rs/bls12_381/0.1.1/bls12_381/notes/serialization/index.html +func (g *G2) FromUncompressed(uncompressed []byte) (*PointG2, error) { + if len(uncompressed) != 4*fpByteSize { + return nil, errors.New("input string length must be equal to 192 bytes") + } + var in [4 * fpByteSize]byte + copy(in[:], uncompressed[:4*fpByteSize]) + if in[0]&(1<<7) != 0 { + return nil, errors.New("compression flag must be zero") + } + if in[0]&(1<<5) != 0 { + return nil, errors.New("sort flag must be zero") + } + if in[0]&(1<<6) != 0 { + for i, v := range in { + if (i == 0 && v != 0x40) || (i != 0 && v != 0x00) { + return nil, errors.New("input string must be zero when infinity flag is set") + } + } + return g.Zero(), nil + } + in[0] &= 0x1f + x, err := g.f.fromBytes(in[:2*fpByteSize]) if err != nil { return nil, err } - p1, err := g.f.fromBytes(in[96:]) + y, err := g.f.fromBytes(in[2*fpByteSize:]) + if err != nil { + return nil, err + } + z := new(fe2).one() + p := &PointG2{*x, *y, *z} + if !g.IsOnCurve(p) { + return nil, errors.New("point is not on curve") + } + if !g.InCorrectSubgroup(p) { + return nil, errors.New("point is not on correct subgroup") + } + return p, nil +} + +// ToUncompressed given a G2 point returns bytes in uncompressed (x, y) form of the point. +// Serialization rules are in line with zcash library. See below for details. +// https://github.com/zcash/librustzcash/blob/master/pairing/src/bls12_381/README.md#serialization +// https://docs.rs/bls12_381/0.1.1/bls12_381/notes/serialization/index.html +func (g *G2) ToUncompressed(p *PointG2) []byte { + out := make([]byte, 4*fpByteSize) + g.Affine(p) + if g.IsZero(p) { + out[0] |= 1 << 6 + return out + } + copy(out[:2*fpByteSize], g.f.toBytes(&p[0])) + copy(out[2*fpByteSize:], g.f.toBytes(&p[1])) + return out +} + +// FromCompressed expects byte slice at least 96 bytes and given bytes returns a new point in G2. +// Serialization rules are in line with zcash library. See below for details. +// https://github.com/zcash/librustzcash/blob/master/pairing/src/bls12_381/README.md#serialization +// https://docs.rs/bls12_381/0.1.1/bls12_381/notes/serialization/index.html +func (g *G2) FromCompressed(compressed []byte) (*PointG2, error) { + if len(compressed) != 2*fpByteSize { + return nil, errors.New("input string length must be equal to 96 bytes") + } + var in [2 * fpByteSize]byte + copy(in[:], compressed[:]) + if in[0]&(1<<7) == 0 { + return nil, errors.New("compression flag must be set") + } + if in[0]&(1<<6) != 0 { + // in[0] == (1 << 6) + (1 << 7) + for i, v := range in { + if (i == 0 && v != 0xc0) || (i != 0 && v != 0x00) { + return nil, errors.New("input string must be zero when infinity flag is set") + } + } + return g.Zero(), nil + } + a := in[0]&(1<<5) != 0 + in[0] &= 0x1f + x, err := g.f.fromBytes(in[:]) + if err != nil { + return nil, err + } + // solve curve equation + y := &fe2{} + g.f.square(y, x) + g.f.mul(y, y, x) + fp2Add(y, y, b2) + if ok := g.f.sqrt(y, y); !ok { + return nil, errors.New("point is not on curve") + } + if y.signBE() == a { + fp2Neg(y, y) + } + z := new(fe2).one() + p := &PointG2{*x, *y, *z} + if !g.InCorrectSubgroup(p) { + return nil, errors.New("point is not on correct subgroup") + } + return p, nil +} + +// ToCompressed given a G2 point returns bytes in compressed form of the point. +// Serialization rules are in line with zcash library. See below for details. +// https://github.com/zcash/librustzcash/blob/master/pairing/src/bls12_381/README.md#serialization +// https://docs.rs/bls12_381/0.1.1/bls12_381/notes/serialization/index.html +func (g *G2) ToCompressed(p *PointG2) []byte { + out := make([]byte, 2*fpByteSize) + g.Affine(p) + if g.IsZero(p) { + out[0] |= 1 << 6 + } else { + copy(out[:], g.f.toBytes(&p[0])) + if !p[1].signBE() { + out[0] |= 1 << 5 + } + } + out[0] |= 1 << 7 + return out +} + +func (g *G2) fromBytesUnchecked(in []byte) (*PointG2, error) { + p0, err := g.f.fromBytes(in[:2*fpByteSize]) + if err != nil { + return nil, err + } + p1, err := g.f.fromBytes(in[2*fpByteSize:]) if err != nil { return nil, err } @@ -93,19 +206,17 @@ func (g *G2) fromBytesUnchecked(in []byte) (*PointG2, error) { } // FromBytes constructs a new point given uncompressed byte input. -// FromBytes does not take zcash flags into account. -// Byte input expected to be larger than 96 bytes. -// First 192 bytes should be concatenation of x and y values +// Input string expected to be 192 bytes and concatenation of x and y values // Point (0, 0) is considered as infinity. func (g *G2) FromBytes(in []byte) (*PointG2, error) { - if len(in) != 192 { - return nil, errors.New("input string should be equal or larger than 192") + if len(in) != 4*fpByteSize { + return nil, errors.New("input string length must be equal to 192 bytes") } - p0, err := g.f.fromBytes(in[:96]) + p0, err := g.f.fromBytes(in[:2*fpByteSize]) if err != nil { return nil, err } - p1, err := g.f.fromBytes(in[96:]) + p1, err := g.f.fromBytes(in[2*fpByteSize:]) if err != nil { return nil, err } @@ -121,60 +232,16 @@ func (g *G2) FromBytes(in []byte) (*PointG2, error) { return p, nil } -// DecodePoint given encoded (x, y) coordinates in 256 bytes returns a valid G2 Point. -func (g *G2) DecodePoint(in []byte) (*PointG2, error) { - if len(in) != 256 { - return nil, errors.New("invalid g2 point length") - } - pointBytes := make([]byte, 192) - x0Bytes, err := decodeFieldElement(in[:64]) - if err != nil { - return nil, err - } - x1Bytes, err := decodeFieldElement(in[64:128]) - if err != nil { - return nil, err - } - y0Bytes, err := decodeFieldElement(in[128:192]) - if err != nil { - return nil, err - } - y1Bytes, err := decodeFieldElement(in[192:]) - if err != nil { - return nil, err - } - copy(pointBytes[:48], x1Bytes) - copy(pointBytes[48:96], x0Bytes) - copy(pointBytes[96:144], y1Bytes) - copy(pointBytes[144:192], y0Bytes) - return g.FromBytes(pointBytes) -} - // ToBytes serializes a point into bytes in uncompressed form, -// does not take zcash flags into account, // returns (0, 0) if point is infinity. func (g *G2) ToBytes(p *PointG2) []byte { - out := make([]byte, 192) + out := make([]byte, 4*fpByteSize) if g.IsZero(p) { return out } g.Affine(p) - copy(out[:96], g.f.toBytes(&p[0])) - copy(out[96:], g.f.toBytes(&p[1])) - return out -} - -// EncodePoint encodes a point into 256 bytes. -func (g *G2) EncodePoint(p *PointG2) []byte { - // outRaw is 96 bytes - outRaw := g.ToBytes(p) - out := make([]byte, 256) - // encode x - copy(out[16:16+48], outRaw[48:96]) - copy(out[80:80+48], outRaw[:48]) - // encode y - copy(out[144:144+48], outRaw[144:]) - copy(out[208:208+48], outRaw[96:144]) + copy(out[:2*fpByteSize], g.f.toBytes(&p[0])) + copy(out[2*fpByteSize:], g.f.toBytes(&p[1])) return out } @@ -212,35 +279,32 @@ func (g *G2) Equal(p1, p2 *PointG2) bool { g.f.square(t[1], &p2[2]) g.f.mul(t[2], t[0], &p2[0]) g.f.mul(t[3], t[1], &p1[0]) - g.f.mul(t[0], t[0], &p1[2]) - g.f.mul(t[1], t[1], &p2[2]) - g.f.mul(t[1], t[1], &p1[1]) - g.f.mul(t[0], t[0], &p2[1]) + g.f.mulAssign(t[0], &p1[2]) + g.f.mulAssign(t[1], &p2[2]) + g.f.mulAssign(t[1], &p1[1]) + g.f.mulAssign(t[0], &p2[1]) return t[0].equal(t[1]) && t[2].equal(t[3]) } -// InCorrectSubgroup checks whether given point is in correct subgroup. -func (g *G2) InCorrectSubgroup(p *PointG2) bool { - tmp := &PointG2{} - g.MulScalar(tmp, p, q) - return g.IsZero(tmp) -} - // IsOnCurve checks a G2 point is on curve. func (g *G2) IsOnCurve(p *PointG2) bool { if g.IsZero(p) { return true } t := g.t - g.f.square(t[0], &p[1]) - g.f.square(t[1], &p[0]) - g.f.mul(t[1], t[1], &p[0]) - g.f.square(t[2], &p[2]) - g.f.square(t[3], t[2]) - g.f.mul(t[2], t[2], t[3]) - g.f.mul(t[2], b2, t[2]) - g.f.add(t[1], t[1], t[2]) - return t[0].equal(t[1]) + g.f.square(t[0], &p[1]) // y^2 + g.f.square(t[1], &p[0]) // x^2 + g.f.mul(t[1], t[1], &p[0]) // x^3 + if p.IsAffine() { + fp2Add(t[1], t[1], b2) // x^2 + b + return t[0].equal(t[1]) // y^2 ?= x^3 + b + } + g.f.square(t[2], &p[2]) // z^2 + g.f.square(t[3], t[2]) // z^4 + g.f.mulAssign(t[2], t[3]) // z^6 + g.f.mulAssign(t[2], b2) // b*z^6 + fp2AddAssign(t[1], t[2]) // x^3 + b * z^6 + return t[0].equal(t[1]) // y^2 ?= x^3 + b * z^6 } // IsAffine checks a G2 point whether it is in affine form. @@ -250,24 +314,102 @@ func (g *G2) IsAffine(p *PointG2) bool { // Affine calculates affine form of given G2 point. func (g *G2) Affine(p *PointG2) *PointG2 { + return g.affine(p, p) +} + +func (g *G2) affine(r, p *PointG2) *PointG2 { if g.IsZero(p) { - return p + return r.Zero() } if !g.IsAffine(p) { t := g.t - g.f.inverse(t[0], &p[2]) - g.f.square(t[1], t[0]) - g.f.mul(&p[0], &p[0], t[1]) - g.f.mul(t[0], t[0], t[1]) - g.f.mul(&p[1], &p[1], t[0]) - p[2].one() + g.f.inverse(t[0], &p[2]) // z^-1 + g.f.square(t[1], t[0]) // z^-2 + g.f.mulAssign(&r[0], t[1]) // x = x * z^-2 + g.f.mulAssign(t[0], t[1]) // z^-3 + g.f.mulAssign(&r[1], t[0]) // y = y * z^-3 + r[2].one() // z = 1 + } else { + r.Set(p) + } + return r +} + +// AffineBatch given multiple of points returns affine representations +func (g *G2) AffineBatch(p []*PointG2) { + inverses := make([]fe2, len(p)) + for i := 0; i < len(p); i++ { + inverses[i].set(&p[i][2]) + } + g.f.inverseBatch(inverses) + t := g.t + for i := 0; i < len(p); i++ { + if !g.IsAffine(p[i]) && !g.IsZero(p[i]) { + g.f.square(t[1], &inverses[i]) + g.f.mulAssign(&p[i][0], t[1]) + g.f.mul(t[0], &inverses[i], t[1]) + g.f.mulAssign(&p[i][1], t[0]) + p[i][2].one() + } } - return p } // Add adds two G2 points p1, p2 and assigns the result to point at first argument. func (g *G2) Add(r, p1, p2 *PointG2) *PointG2 { - // http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-add-2007-bl + // http://www.hyperelliptic.org/EFD/gp/auto-shortw-jacobian-0.html#addition-add-2007-bl + if g.IsZero(p1) { + return r.Set(p2) + } + if g.IsZero(p2) { + return r.Set(p1) + } + if g.IsAffine(p2) { + return g.AddMixed(r, p1, p2) + } + t := g.t + g.f.square(t[7], &p1[2]) // z1z1 + g.f.mul(t[1], &p2[0], t[7]) // u2 = x2 * z1z1 + g.f.mul(t[2], &p1[2], t[7]) // z1z1 * z1 + g.f.mul(t[0], &p2[1], t[2]) // s2 = y2 * z1z1 * z1 + g.f.square(t[8], &p2[2]) // z2z2 + g.f.mul(t[3], &p1[0], t[8]) // u1 = x1 * z2z2 + g.f.mul(t[4], &p2[2], t[8]) // z2z2 * z2 + g.f.mul(t[2], &p1[1], t[4]) // s1 = y1 * z2z2 * z2 + if t[1].equal(t[3]) { + if t[0].equal(t[2]) { + return g.Double(r, p1) + } else { + return r.Zero() + } + } + fp2SubAssign(t[1], t[3]) // h = u2 - u1 + fp2Double(t[4], t[1]) // 2h + g.f.squareAssign(t[4]) // i = 2h^2 + g.f.mul(t[5], t[1], t[4]) // j = h*i + fp2SubAssign(t[0], t[2]) // s2 - s1 + fp2DoubleAssign(t[0]) // r = 2*(s2 - s1) + g.f.square(t[6], t[0]) // r^2 + fp2SubAssign(t[6], t[5]) // r^2 - j + g.f.mulAssign(t[3], t[4]) // v = u1 * i + fp2Double(t[4], t[3]) // 2*v + fp2Sub(&r[0], t[6], t[4]) // x3 = r^2 - j - 2*v + fp2Sub(t[4], t[3], &r[0]) // v - x3 + g.f.mul(t[6], t[2], t[5]) // s1 * j + fp2DoubleAssign(t[6]) // 2 * s1 * j + g.f.mulAssign(t[0], t[4]) // r * (v - x3) + fp2Sub(&r[1], t[0], t[6]) // y3 = r * (v - x3) - (2 * s1 * j) + fp2Add(t[0], &p1[2], &p2[2]) // z1 + z2 + g.f.squareAssign(t[0]) // (z1 + z2)^2 + fp2SubAssign(t[0], t[7]) // (z1 + z2)^2 - z1z1 + fp2SubAssign(t[0], t[8]) // (z1 + z2)^2 - z1z1 - z2z2 + g.f.mul(&r[2], t[0], t[1]) // z3 = ((z1 + z2)^2 - z1z1 - z2z2) * h + return r +} + +// Add adds two G1 points p1, p2 and assigns the result to point at first argument. +// Expects the second point p2 in affine form. +func (g *G2) AddMixed(r, p1, p2 *PointG2) *PointG2 { + // http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#addition-madd-2007-bl if g.IsZero(p1) { return r.Set(p2) } @@ -275,80 +417,75 @@ func (g *G2) Add(r, p1, p2 *PointG2) *PointG2 { return r.Set(p1) } t := g.t - g.f.square(t[7], &p1[2]) - g.f.mul(t[1], &p2[0], t[7]) - g.f.mul(t[2], &p1[2], t[7]) - g.f.mul(t[0], &p2[1], t[2]) - g.f.square(t[8], &p2[2]) - g.f.mul(t[3], &p1[0], t[8]) - g.f.mul(t[4], &p2[2], t[8]) - g.f.mul(t[2], &p1[1], t[4]) - if t[1].equal(t[3]) { - if t[0].equal(t[2]) { - return g.Double(r, p1) - } - return r.Zero() + g.f.square(t[7], &p1[2]) // z1z1 + g.f.mul(t[1], &p2[0], t[7]) // u2 = x2 * z1z1 + g.f.mul(t[2], &p1[2], t[7]) // z1z1 * z1 + g.f.mul(t[0], &p2[1], t[2]) // s2 = y2 * z1z1 * z1 + + if p1[0].equal(t[1]) && p1[1].equal(t[0]) { + return g.Double(r, p1) } - g.f.sub(t[1], t[1], t[3]) - g.f.double(t[4], t[1]) - g.f.square(t[4], t[4]) - g.f.mul(t[5], t[1], t[4]) - g.f.sub(t[0], t[0], t[2]) - g.f.double(t[0], t[0]) - g.f.square(t[6], t[0]) - g.f.sub(t[6], t[6], t[5]) - g.f.mul(t[3], t[3], t[4]) - g.f.double(t[4], t[3]) - g.f.sub(&r[0], t[6], t[4]) - g.f.sub(t[4], t[3], &r[0]) - g.f.mul(t[6], t[2], t[5]) - g.f.double(t[6], t[6]) - g.f.mul(t[0], t[0], t[4]) - g.f.sub(&r[1], t[0], t[6]) - g.f.add(t[0], &p1[2], &p2[2]) - g.f.square(t[0], t[0]) - g.f.sub(t[0], t[0], t[7]) - g.f.sub(t[0], t[0], t[8]) - g.f.mul(&r[2], t[0], t[1]) + + fp2SubAssign(t[1], &p1[0]) // h = u2 - x1 + g.f.square(t[2], t[1]) // hh + fp2Double(t[4], t[2]) + fp2DoubleAssign(t[4]) // 4hh + g.f.mul(t[5], t[1], t[4]) // j = h*i + fp2SubAssign(t[0], &p1[1]) // s2 - y1 + fp2DoubleAssign(t[0]) // r = 2*(s2 - y1) + g.f.square(t[6], t[0]) // r^2 + fp2SubAssign(t[6], t[5]) // r^2 - j + g.f.mul(t[3], &p1[0], t[4]) // v = x1 * i + fp2Double(t[4], t[3]) // 2*v + fp2Sub(&r[0], t[6], t[4]) // x3 = r^2 - j - 2*v + fp2Sub(t[4], t[3], &r[0]) // v - x3 + g.f.mul(t[6], &p1[1], t[5]) // y1 * j + fp2DoubleAssign(t[6]) // 2 * y1 * j + g.f.mulAssign(t[0], t[4]) // r * (v - x3) + fp2Sub(&r[1], t[0], t[6]) // y3 = r * (v - x3) - (2 * y1 * j) + fp2Add(t[0], &p1[2], t[1]) // z1 + h + g.f.squareAssign(t[0]) // (z1 + h)^2 + fp2SubAssign(t[0], t[7]) // (z1 + h)^2 - z1z1 + fp2Sub(&r[2], t[0], t[2]) // z3 = (z1 + z2)^2 - z1z1 - hh return r } // Double doubles a G2 point p and assigns the result to the point at first argument. func (g *G2) Double(r, p *PointG2) *PointG2 { - // http://www.hyperelliptic.org/EFD/g1p/auto-shortw-jacobian-0.html#doubling-dbl-2009-l + // http://www.hyperelliptic.org/EFD/gp/auto-shortw-jacobian-0.html#doubling-dbl-2009-l if g.IsZero(p) { return r.Set(p) } t := g.t - g.f.square(t[0], &p[0]) - g.f.square(t[1], &p[1]) - g.f.square(t[2], t[1]) - g.f.add(t[1], &p[0], t[1]) - g.f.square(t[1], t[1]) - g.f.sub(t[1], t[1], t[0]) - g.f.sub(t[1], t[1], t[2]) - g.f.double(t[1], t[1]) - g.f.double(t[3], t[0]) - g.f.add(t[0], t[3], t[0]) - g.f.square(t[4], t[0]) - g.f.double(t[3], t[1]) - g.f.sub(&r[0], t[4], t[3]) - g.f.sub(t[1], t[1], &r[0]) - g.f.double(t[2], t[2]) - g.f.double(t[2], t[2]) - g.f.double(t[2], t[2]) - g.f.mul(t[0], t[0], t[1]) - g.f.sub(t[1], t[0], t[2]) - g.f.mul(t[0], &p[1], &p[2]) - r[1].set(t[1]) - g.f.double(&r[2], t[0]) + g.f.square(t[0], &p[0]) // a = x^2 + g.f.square(t[1], &p[1]) // b = y^2 + g.f.square(t[2], t[1]) // c = b^2 + fp2AddAssign(t[1], &p[0]) // b + x1 + g.f.squareAssign(t[1]) // (b + x1)^2 + fp2SubAssign(t[1], t[0]) // (b + x1)^2 - a + fp2SubAssign(t[1], t[2]) // (b + x1)^2 - a - c + fp2DoubleAssign(t[1]) // d = 2((b+x1)^2 - a - c) + fp2Double(t[3], t[0]) // 2a + fp2AddAssign(t[0], t[3]) // e = 3a + g.f.square(t[4], t[0]) // f = e^2 + fp2Double(t[3], t[1]) // 2d + fp2Sub(&r[0], t[4], t[3]) // x3 = f - 2d + fp2SubAssign(t[1], &r[0]) // d-x3 + fp2DoubleAssign(t[2]) // + fp2DoubleAssign(t[2]) // + fp2DoubleAssign(t[2]) // 8c + g.f.mulAssign(t[0], t[1]) // e * (d - x3) + fp2Sub(t[1], t[0], t[2]) // x3 = e * (d - x3) - 8c + g.f.mul(t[0], &p[1], &p[2]) // y1 * z1 + r[1].set(t[1]) // + fp2Double(&r[2], t[0]) // z3 = 2(y1 * z1) return r } // Neg negates a G2 point p and assigns the result to the point at first argument. func (g *G2) Neg(r, p *PointG2) *PointG2 { r[0].set(&p[0]) - g.f.neg(&r[1], &p[1]) + fp2Neg(&r[1], &p[1]) r[2].set(&p[2]) return r } @@ -361,8 +498,29 @@ func (g *G2) Sub(c, a, b *PointG2) *PointG2 { return c } -// MulScalar multiplies a point by given scalar value in big.Int and assigns the result to point at first argument. -func (g *G2) MulScalar(c, p *PointG2, e *big.Int) *PointG2 { +// MulScalar multiplies a point by given scalar value and assigns the result to point at first argument. +func (g *G2) MulScalar(r, p *PointG2, e *Fr) *PointG2 { + return g.glvMulFr(r, p, e) +} + +// MulScalarBig multiplies a point by given scalar value in big.Int and assigns the result to point at first argument. +func (g *G2) MulScalarBig(r, p *PointG2, e *big.Int) *PointG2 { + return g.glvMulBig(r, p, e) +} + +func (g *G2) mulScalar(c, p *PointG2, e *Fr) *PointG2 { + q, n := &PointG2{}, &PointG2{} + n.Set(p) + for i := 0; i < frBitSize; i++ { + if e.Bit(i) { + g.Add(q, q, n) + } + g.Double(n, n) + } + return c.Set(q) +} + +func (g *G2) mulScalarBig(c, p *PointG2, e *big.Int) *PointG2 { q, n := &PointG2{}, &PointG2{} n.Set(p) l := e.BitLen() @@ -375,67 +533,299 @@ func (g *G2) MulScalar(c, p *PointG2, e *big.Int) *PointG2 { return c.Set(q) } -// ClearCofactor maps given a G2 point to correct subgroup -func (g *G2) ClearCofactor(p *PointG2) { - g.MulScalar(p, p, cofactorEFFG2) +func (g *G2) wnafMulFr(r, p *PointG2, e *Fr) *PointG2 { + wnaf := e.toWNAF(wnafMulWindowG2) + return g.wnafMul(r, p, wnaf) } -// MultiExp calculates multi exponentiation. Given pairs of G2 point and scalar values -// (P_0, e_0), (P_1, e_1), ... (P_n, e_n) calculates r = e_0 * P_0 + e_1 * P_1 + ... + e_n * P_n -// Length of points and scalars are expected to be equal, otherwise an error is returned. -// Result is assigned to point at first argument. -func (g *G2) MultiExp(r *PointG2, points []*PointG2, powers []*big.Int) (*PointG2, error) { - if len(points) != len(powers) { - return nil, errors.New("point and scalar vectors should be in same length") +func (g *G2) wnafMulBig(r, p *PointG2, e *big.Int) *PointG2 { + wnaf := bigToWNAF(e, wnafMulWindowG2) + return g.wnafMul(r, p, wnaf) +} + +func (g *G2) wnafMul(c, p *PointG2, wnaf nafNumber) *PointG2 { + + l := (1 << (wnafMulWindowG2 - 1)) + + twoP, acc := g.New(), new(PointG2).Set(p) + g.Double(twoP, p) + g.Affine(twoP) + + // table = {p, 3p, 5p, ..., -p, -3p, -5p} + table := make([]*PointG2, l*2) + table[0], table[l] = g.New(), g.New() + table[0].Set(p) + g.Neg(table[l], table[0]) + + for i := 1; i < l; i++ { + g.AddMixed(acc, acc, twoP) + table[i], table[i+l] = g.New(), g.New() + table[i].Set(acc) + g.Neg(table[i+l], table[i]) } - var c uint32 = 3 - if len(powers) >= 32 { - c = uint32(math.Ceil(math.Log10(float64(len(powers))))) - } - bucketSize, numBits := (1<= 0; i-- { + if wnaf[i] > 0 { + g.Add(q, q, table[wnaf[i]>>1]) + } else if wnaf[i] < 0 { + g.Add(q, q, table[((-wnaf[i])>>1)+l]) } - for i := 0; i < len(powers); i++ { - s0 := powers[i].Uint64() - index := uint(s0 & mask) - if index != 0 { - g.Add(bucket[index-1], bucket[index-1], points[i]) + if i != 0 { + g.Double(q, q) + } + } + return c.Set(q) +} + +func (g *G2) glvMulFr(r, p *PointG2, e *Fr) *PointG2 { + return g.glvMul(r, p, new(glvVectorFr).new(e)) +} + +func (g *G2) glvMulBig(r, p *PointG2, e *big.Int) *PointG2 { + return g.glvMul(r, p, new(glvVectorBig).new(e)) +} + +func (g *G2) glvMul(r, p0 *PointG2, v glvVector) *PointG2 { + + w := glvMulWindowG2 + l := 1 << (w - 1) + + // prepare tables + // tableK1 = {P, 3P, 5P, ...} + // tableK2 = {λP, 3λP, 5λP, ...} + tableK1, tableK2 := make([]*PointG2, l), make([]*PointG2, l) + double := g.New() + g.Double(double, p0) + g.affine(double, double) + tableK1[0] = new(PointG2) + tableK1[0].Set(p0) + for i := 1; i < l; i++ { + tableK1[i] = new(PointG2) + g.AddMixed(tableK1[i], tableK1[i-1], double) + } + g.AffineBatch(tableK1) + for i := 0; i < l; i++ { + tableK2[i] = new(PointG2) + g.glvEndomorphism(tableK2[i], tableK1[i]) + } + + // recode small scalars + naf1, naf2 := v.wnaf(w) + lenNAF1, lenNAF2 := len(naf1), len(naf2) + lenNAF := lenNAF1 + if lenNAF2 > lenNAF { + lenNAF = lenNAF2 + } + + acc, p1 := g.New(), g.New() + + // function for naf addition + add := func(table []*PointG2, naf int) { + if naf != 0 { + nafAbs := naf + if nafAbs < 0 { + nafAbs = -nafAbs } - powers[i] = new(big.Int).Rsh(powers[i], uint(c)) + p1.Set(table[nafAbs>>1]) + if naf < 0 { + g.Neg(p1, p1) + } + g.AddMixed(acc, acc, p1) } - sum.Zero() - for i := len(bucket) - 1; i >= 0; i-- { - g.Add(sum, sum, bucket[i]) - g.Add(acc, acc, sum) - } - windows[j] = g.New() - windows[j].Set(acc) - j++ - cur += c } - acc.Zero() - for i := len(windows) - 1; i >= 0; i-- { - for j := uint32(0); j < c; j++ { + + // sliding + for i := lenNAF - 1; i >= 0; i-- { + if i < lenNAF1 { + add(tableK1, naf1[i]) + } + if i < lenNAF2 { + add(tableK2, naf2[i]) + } + if i != 0 { g.Double(acc, acc) } - g.Add(acc, acc, windows[i]) + } + return r.Set(acc) +} + +// MultiExpBig calculates multi exponentiation. Scalar values are received as big.Int type. +// Given pairs of G2 point and scalar values `(P_0, e_0), (P_1, e_1), ... (P_n, e_n)`, +// calculates `r = e_0 * P_0 + e_1 * P_1 + ... + e_n * P_n`. +// Length of points and scalars are expected to be equal, otherwise an error is returned. +// Result is assigned to point at first argument. +func (g *G2) MultiExpBig(r *PointG2, points []*PointG2, scalars []*big.Int) (*PointG2, error) { + if len(points) != len(scalars) { + return nil, errors.New("point and scalar vectors should be in same length") + } + + c := 3 + if len(scalars) >= 32 { + c = int(math.Ceil(math.Log(float64(len(scalars))))) + } + + bucketSize := (1 << c) - 1 + windows := make([]PointG2, 255/c+1) + bucket := make([]PointG2, bucketSize) + + for j := 0; j < len(windows); j++ { + + for i := 0; i < bucketSize; i++ { + bucket[i].Zero() + } + + for i := 0; i < len(scalars); i++ { + index := bucketSize & int(new(big.Int).Rsh(scalars[i], uint(c*j)).Int64()) + if index != 0 { + g.Add(&bucket[index-1], &bucket[index-1], points[i]) + } + } + + acc, sum := g.New(), g.New() + for i := bucketSize - 1; i >= 0; i-- { + g.Add(sum, sum, &bucket[i]) + g.Add(acc, acc, sum) + } + windows[j].Set(acc) + } + + acc := g.New() + for i := len(windows) - 1; i >= 0; i-- { + for j := 0; j < c; j++ { + g.Double(acc, acc) + } + g.Add(acc, acc, &windows[i]) } return r.Set(acc), nil } +// MultiExp calculates multi exponentiation. Given pairs of G2 point and scalar values `(P_0, e_0), (P_1, e_1), ... (P_n, e_n)`, +// calculates `r = e_0 * P_0 + e_1 * P_1 + ... + e_n * P_n`. Length of points and scalars are expected to be equal, +// otherwise an error is returned. Result is assigned to point at first argument. +func (g *G2) MultiExp(r *PointG2, points []*PointG2, scalars []*Fr) (*PointG2, error) { + if len(points) != len(scalars) { + return nil, errors.New("point and scalar vectors should be in same length") + } + + g.AffineBatch(points) + + c := 3 + if len(scalars) >= 32 { + c = int(math.Ceil(math.Log(float64(len(scalars))))) + } + + bucketSize := (1 << c) - 1 + windows := make([]*PointG2, 255/c+1) + bucket := make([]PointG2, bucketSize) + + for j := 0; j < len(windows); j++ { + + for i := 0; i < bucketSize; i++ { + bucket[i].Zero() + } + + for i := 0; i < len(scalars); i++ { + index := bucketSize & int(scalars[i].sliceUint64(c*j)) + if index != 0 { + g.AddMixed(&bucket[index-1], &bucket[index-1], points[i]) + } + } + + acc, sum := g.New(), g.New() + for i := bucketSize - 1; i >= 0; i-- { + g.Add(sum, sum, &bucket[i]) + g.Add(acc, acc, sum) + } + windows[j] = g.New().Set(acc) + } + + g.AffineBatch(windows) + + acc := g.New() + for i := len(windows) - 1; i >= 0; i-- { + for j := 0; j < c; j++ { + g.Double(acc, acc) + } + g.AddMixed(acc, acc, windows[i]) + } + return r.Set(acc), nil +} + +// InCorrectSubgroup checks whether given point is in correct subgroup. +func (g *G2) InCorrectSubgroup(p *PointG2) bool { + + // Faster Subgroup Checks for BLS12-381 + // S. Bowe + // https://eprint.iacr.org/2019/814.pdf + + // [z]ψ^3(P) − ψ^2(P) + P = O + t0, t1 := g.New().Set(p), g.New() + + g.psi(t0) + g.psi(t0) + g.Neg(t1, t0) // - ψ^2(P) + g.psi(t0) // ψ^3(P) + g.mulX(t0) // - x ψ^3(P) + g.Neg(t0, t0) + + g.Add(t0, t0, t1) + g.Add(t0, t0, p) + + return g.IsZero(t0) +} + +// ClearCofactor maps given a G2 point to correct subgroup +func (g *G2) ClearCofactor(p *PointG2) *PointG2 { + + // Efficient hash maps to G2 on BLS curves + // A. Budroni, F. Pintore + // https://eprint.iacr.org/2017/419.pdf + + // [h(ψ)]P = [x^2 − x − 1]P + [x − 1]ψ(P) + ψ^2(2P) + t0, t1, t2, t3 := g.New().Set(p), g.New().Set(p), g.New().Set(p), g.New() + + g.Double(t0, t0) + g.psi(t0) + g.psi(t0) // P2 = ψ^2(2P) + g.psi(t2) // P1 = ψ(P) + g.mulX(t1) // -xP0 + + g.Sub(t3, t1, t2) // -xP0 - P1 + g.mulX(t3) // (x^2)P0 + xP1 + g.Sub(t1, t1, p) // (-x-1)P0 + g.Add(t3, t3, t1) // (x^2-x-1)P0 + xP1 + g.Sub(t3, t3, t2) // (x^2-x-1)P0 + (x-1)P1 + g.Add(t3, t3, t0) // (x^2-x-1)P0 + (x-1)P1 + P2 + return p.Set(t3) +} + +func (g *G2) psi(p *PointG2) { + fp2Conjugate(&p[0], &p[0]) + fp2Conjugate(&p[1], &p[1]) + fp2Conjugate(&p[2], &p[2]) + g.f.mul(&p[0], &p[0], &psix) + g.f.mul(&p[1], &p[1], &psiy) +} + +func (g *G2) mulX(p *PointG2) { + + chain := func(p0 *PointG2, n int, p1 *PointG2) { + g.Add(p0, p0, p1) + for i := 0; i < n; i++ { + g.Double(p0, p0) + } + } + + t := g.New().Set(p) + g.Double(p, t) + chain(p, 2, t) + chain(p, 3, t) + chain(p, 9, t) + chain(p, 32, t) + chain(p, 16, t) +} + // MapToCurve given a byte slice returns a valid G2 point. // This mapping function implements the Simplified Shallue-van de Woestijne-Ulas method. // https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-05#section-6.6.2 @@ -453,3 +843,45 @@ func (g *G2) MapToCurve(in []byte) (*PointG2, error) { g.ClearCofactor(q) return g.Affine(q), nil } + +// EncodeToCurve given a message and domain seperator tag returns the hash result +// which is a valid curve point. +// Implementation follows BLS12381G1_XMD:SHA-256_SSWU_NU_ suite at +// https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06 +func (g *G2) EncodeToCurve(msg, domain []byte) (*PointG2, error) { + hashRes, err := hashToFpXMDSHA256(msg, domain, 2) + if err != nil { + return nil, err + } + fp2 := g.f + u := &fe2{*hashRes[0], *hashRes[1]} + x, y := swuMapG2(fp2, u) + isogenyMapG2(fp2, x, y) + z := new(fe2).one() + q := &PointG2{*x, *y, *z} + g.ClearCofactor(q) + return g.Affine(q), nil +} + +// HashToCurve given a message and domain seperator tag returns the hash result +// which is a valid curve point. +// Implementation follows BLS12381G1_XMD:SHA-256_SSWU_RO_ suite at +// https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06 +func (g *G2) HashToCurve(msg, domain []byte) (*PointG2, error) { + hashRes, err := hashToFpXMDSHA256(msg, domain, 4) + if err != nil { + return nil, err + } + fp2 := g.f + u0, u1 := &fe2{*hashRes[0], *hashRes[1]}, &fe2{*hashRes[2], *hashRes[3]} + x0, y0 := swuMapG2(fp2, u0) + x1, y1 := swuMapG2(fp2, u1) + z0 := new(fe2).one() + z1 := new(fe2).one() + p0, p1 := &PointG2{*x0, *y0, *z0}, &PointG2{*x1, *y1, *z1} + g.Add(p0, p0, p1) + g.Affine(p0) + isogenyMapG2(fp2, &p0[0], &p0[1]) + g.ClearCofactor(p0) + return g.Affine(p0), nil +} diff --git a/crypto/bls12381/g2_test.go b/crypto/bls12381/g2_test.go index 4d1f3a19ac..86cbd3f619 100644 --- a/crypto/bls12381/g2_test.go +++ b/crypto/bls12381/g2_test.go @@ -3,54 +3,117 @@ package bls12381 import ( "bytes" "crypto/rand" + "fmt" + "io/ioutil" "math/big" "testing" - - "github.com/ethereum/go-ethereum/common" ) func (g *G2) one() *PointG2 { - one, _ := g.fromBytesUnchecked( - common.FromHex("" + - "13e02b6052719f607dacd3a088274f65596bd0d09920b61ab5da61bbdc7f5049334cf11213945d57e5ac7d055d042b7e" + - "024aa2b2f08f0a91260805272dc51051c6e47ad4fa403b02b4510b647ae3d1770bac0326a805bbefd48056c8c121bdb8" + - "0606c4a02ea734cc32acd2b02bc28b99cb3e287e85a763af267492ab572e99ab3f370d275cec1da1aaa9075ff05f79be" + - "0ce5d527727d6e118cc9cdc6da2e351aadfd9baa8cbdd3a76d429a695160d12c923ac9cc3baca289e193548608b82801", - ), - ) - return one + return g.New().Set(&g2One) } func (g *G2) rand() *PointG2 { - k, err := rand.Int(rand.Reader, q) - if err != nil { - panic(err) + p := &PointG2{} + z, _ := new(fe2).rand(rand.Reader) + z6, bz6 := new(fe2), new(fe2) + g.f.square(z6, z) + g.f.square(z6, z6) + g.f.mul(z6, z6, z) + g.f.mul(z6, z6, z) + g.f.mul(bz6, z6, b2) + for { + x, _ := new(fe2).rand(rand.Reader) + y := new(fe2) + g.f.square(y, x) + g.f.mul(y, y, x) + fp2Add(y, y, bz6) + if g.f.sqrt(y, y) { + p.Set(&PointG2{*x, *y, *z}) + break + } } - return g.MulScalar(&PointG2{}, g.one(), k) + if !g.IsOnCurve(p) { + panic("rand point must be on curve") + } + if g.InCorrectSubgroup(p) { + panic("rand point must be out of correct subgroup") + } + return p +} + +func (g *G2) randCorrect() *PointG2 { + p := g.ClearCofactor(g.rand()) + if !g.InCorrectSubgroup(p) { + panic("must be in correct subgroup") + } + return p +} + +func (g *G2) randAffine() *PointG2 { + return g.Affine(g.randCorrect()) +} + +func (g *G2) new() *PointG2 { + return g.Zero() } func TestG2Serialization(t *testing.T) { + var err error g2 := NewG2() + zero := g2.Zero() + b0 := g2.ToUncompressed(zero) + p0, err := g2.FromUncompressed(b0) + if err != nil { + t.Fatal(err) + } + if !g2.IsZero(p0) { + t.Fatal("infinity serialization failed") + } + b0 = g2.ToCompressed(zero) + p0, err = g2.FromCompressed(b0) + if err != nil { + t.Fatal(err) + } + if !g2.IsZero(p0) { + t.Fatal("infinity serialization failed") + } + b0 = g2.ToBytes(zero) + p0, err = g2.FromBytes(b0) + if err != nil { + t.Fatal(err) + } + if !g2.IsZero(p0) { + t.Fatal("infinity serialization failed") + } for i := 0; i < fuz; i++ { - a := g2.rand() - buf := g2.ToBytes(a) - b, err := g2.FromBytes(buf) + a := g2.randAffine() + uncompressed := g2.ToUncompressed(a) + b, err := g2.FromUncompressed(uncompressed) if err != nil { t.Fatal(err) } if !g2.Equal(a, b) { - t.Fatal("bad serialization from/to") + t.Fatal("serialization failed") + } + compressed := g2.ToCompressed(b) + a, err = g2.FromCompressed(compressed) + if err != nil { + t.Fatal(err) + } + if !g2.Equal(a, b) { + t.Fatal("serialization failed") } } for i := 0; i < fuz; i++ { a := g2.rand() - encoded := g2.EncodePoint(a) - b, err := g2.DecodePoint(encoded) + uncompressed := g2.ToBytes(a) + b, err := g2.FromBytes(uncompressed) if err != nil { t.Fatal(err) } if !g2.Equal(a, b) { - t.Fatal("bad serialization encode/decode") + t.Fatal("serialization failed") } } } @@ -68,6 +131,26 @@ func TestG2IsOnCurve(t *testing.T) { } } +func TestG2BatchAffine(t *testing.T) { + n := 20 + g := NewG2() + points0 := make([]*PointG2, n) + points1 := make([]*PointG2, n) + for i := 0; i < n; i++ { + points0[i] = g.rand() + points1[i] = g.New().Set(points0[i]) + if g.IsAffine(points0[i]) { + t.Fatal("expect non affine point") + } + } + g.AffineBatch(points0) + for i := 0; i < n; i++ { + if !g.Equal(points0[i], points1[i]) { + t.Fatal("batch affine failed") + } + } +} + func TestG2AdditiveProperties(t *testing.T) { g := NewG2() t0, t1 := g.New(), g.New() @@ -138,14 +221,71 @@ func TestG2AdditiveProperties(t *testing.T) { } } +func TestG2MixedAdd(t *testing.T) { + g := NewG2() + for i := 0; i < fuz; i++ { + a, b := g.rand(), g.rand() + if g.IsAffine(a) || g.IsAffine(b) { + t.Fatal("expect non affine points") + } + bAffine := g.New().Set(b) + g.Affine(bAffine) + r0, r1 := g.New(), g.New() + g.Add(r0, a, b) + g.AddMixed(r1, a, bAffine) + if !g.Equal(r0, r1) { + t.Fatal("mixed addition failed") + } + aAffine := g.New().Set(a) + g.Affine(aAffine) + g.AddMixed(r0, a, aAffine) + g.Double(r1, a) + if !g.Equal(r0, r1) { + t.Fatal("mixed addition must double where points are equal") + } + } +} + +func TestG2MultiplicationCross(t *testing.T) { + g := NewG2() + for i := 0; i < fuz; i++ { + + a := g.randCorrect() + s, _ := new(Fr).Rand(rand.Reader) + sBig := s.ToBig() + res0, res1, res2, res3, res4 := g.New(), g.New(), g.New(), g.New(), g.New() + + g.mulScalar(res0, a, s) + g.glvMulFr(res1, a, s) + g.glvMulBig(res2, a, sBig) + g.wnafMulFr(res3, a, s) + g.wnafMulBig(res4, a, sBig) + + if !g.Equal(res0, res1) { + t.Fatal("cross multiplication failed (glv, fr)", i) + } + if !g.Equal(res0, res2) { + t.Fatal("cross multiplication failed (glv, big)", i) + } + if !g.Equal(res0, res3) { + t.Fatal("cross multiplication failed (wnaf, fr)", i) + } + if !g.Equal(res0, res4) { + t.Fatal("cross multiplication failed (wnaf, big)", i) + } + } +} + func TestG2MultiplicativeProperties(t *testing.T) { g := NewG2() t0, t1 := g.New(), g.New() zero := g.Zero() for i := 0; i < fuz; i++ { - a := g.rand() - s1, s2, s3 := randScalar(q), randScalar(q), randScalar(q) - sone := big.NewInt(1) + a := g.randCorrect() + s1, _ := new(Fr).Rand(rand.Reader) + s2, _ := new(Fr).Rand(rand.Reader) + s3, _ := new(Fr).Rand(rand.Reader) + sone := &Fr{1} g.MulScalar(t0, zero, s1) if !g.Equal(t0, zero) { t.Fatal(" 0 ^ s == 0") @@ -163,7 +303,7 @@ func TestG2MultiplicativeProperties(t *testing.T) { s3.Mul(s1, s2) g.MulScalar(t1, a, s3) if !g.Equal(t0, t1) { - t.Errorf(" (a ^ s1) ^ s2 == a ^ (s1 * s2)") + t.Fatal(" (a ^ s1) ^ s2 == a ^ (s1 * s2)") } g.MulScalar(t0, a, s1) g.MulScalar(t1, a, s2) @@ -171,12 +311,71 @@ func TestG2MultiplicativeProperties(t *testing.T) { s3.Add(s1, s2) g.MulScalar(t1, a, s3) if !g.Equal(t0, t1) { - t.Errorf(" (a ^ s1) + (a ^ s2) == a ^ (s1 + s2)") + t.Fatal(" (a ^ s1) + (a ^ s2) == a ^ (s1 + s2)") } } } +func TestZKCryptoVectorsG2UncompressedValid(t *testing.T) { + data, err := ioutil.ReadFile("tests/g2_uncompressed_valid_test_vectors.dat") + if err != nil { + panic(err) + } + g := NewG2() + p1 := g.Zero() + for i := 0; i < 1000; i++ { + vector := data[i*192 : (i+1)*192] + p2, err := g.FromUncompressed(vector) + if err != nil { + t.Fatal("decoing fails", err, i) + } + uncompressed := g.ToUncompressed(p2) + if !bytes.Equal(vector, uncompressed) || !g.Equal(p1, p2) { + t.Fatal("serialization failed") + } + g.Add(p1, p1, &g2One) + } +} + +func TestZKCryptoVectorsG2CompressedValid(t *testing.T) { + data, err := ioutil.ReadFile("tests/g2_compressed_valid_test_vectors.dat") + if err != nil { + panic(err) + } + g := NewG2() + p1 := g.Zero() + for i := 0; i < 1000; i++ { + vector := data[i*2*fpByteSize : (i+1)*2*fpByteSize] + p2, err := g.FromCompressed(vector) + if err != nil { + t.Fatal("decoing fails", err, i) + } + compressed := g.ToCompressed(p2) + if !bytes.Equal(vector, compressed) || !g.Equal(p1, p2) { + t.Fatal("serialization failed") + } + + g.Add(p1, p1, &g2One) + } +} + func TestG2MultiExpExpected(t *testing.T) { + g := NewG2() + one := g.one() + var scalars [2]*Fr + var bases [2]*PointG2 + scalars[0] = &Fr{2} + scalars[1] = &Fr{3} + bases[0], bases[1] = new(PointG2).Set(one), new(PointG2).Set(one) + expected, result := g.New(), g.New() + g.mulScalar(expected, one, &Fr{5}) + _, _ = g.MultiExp(result, bases[:], scalars[:]) + if !g.Equal(expected, result) { + t.Fatal("multi-exponentiation failed") + } +} + +func TestG2MultiExpBigExpected(t *testing.T) { g := NewG2() one := g.one() var scalars [2]*big.Int @@ -185,36 +384,76 @@ func TestG2MultiExpExpected(t *testing.T) { scalars[1] = big.NewInt(3) bases[0], bases[1] = new(PointG2).Set(one), new(PointG2).Set(one) expected, result := g.New(), g.New() - g.MulScalar(expected, one, big.NewInt(5)) - _, _ = g.MultiExp(result, bases[:], scalars[:]) + g.mulScalarBig(expected, one, big.NewInt(5)) + _, _ = g.MultiExpBig(result, bases[:], scalars[:]) if !g.Equal(expected, result) { - t.Fatal("bad multi-exponentiation") + t.Fatal("multi-exponentiation failed") } } -func TestG2MultiExpBatch(t *testing.T) { +func TestG2MultiExp(t *testing.T) { g := NewG2() - one := g.one() - n := 1000 - bases := make([]*PointG2, n) - scalars := make([]*big.Int, n) - // scalars: [s0,s1 ... s(n-1)] - // bases: [P0,P1,..P(n-1)] = [s(n-1)*G, s(n-2)*G ... s0*G] - for i, j := 0, n-1; i < n; i, j = i+1, j-1 { - scalars[j], _ = rand.Int(rand.Reader, big.NewInt(100000)) - bases[i] = g.New() - g.MulScalar(bases[i], one, scalars[j]) + for n := 1; n < 1024+1; n = n * 2 { + bases := make([]*PointG2, n) + scalars := make([]*Fr, n) + var err error + for i := 0; i < n; i++ { + scalars[i], err = new(Fr).Rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + bases[i] = g.rand() + } + expected, tmp := g.New(), g.New() + for i := 0; i < n; i++ { + g.mulScalar(tmp, bases[i], scalars[i]) + g.Add(expected, expected, tmp) + } + result := g.New() + _, _ = g.MultiExp(result, bases, scalars) + if !g.Equal(expected, result) { + t.Fatal("multi-exponentiation failed") + } } - // expected: s(n-1)*P0 + s(n-2)*P1 + s0*P(n-1) - expected, tmp := g.New(), g.New() - for i := 0; i < n; i++ { - g.MulScalar(tmp, bases[i], scalars[i]) - g.Add(expected, expected, tmp) +} + +func TestG2MultiExpBig(t *testing.T) { + g := NewG2() + for n := 1; n < 1024+1; n = n * 2 { + bases := make([]*PointG2, n) + scalars := make([]*big.Int, n) + var err error + for i := 0; i < n; i++ { + scalars[i], err = rand.Int(rand.Reader, qBig) + if err != nil { + t.Fatal(err) + } + bases[i] = g.rand() + } + expected, tmp := g.New(), g.New() + for i := 0; i < n; i++ { + g.mulScalarBig(tmp, bases[i], scalars[i]) + g.Add(expected, expected, tmp) + } + result := g.New() + _, _ = g.MultiExpBig(result, bases, scalars) + if !g.Equal(expected, result) { + t.Fatal("multi-exponentiation failed") + } } - result := g.New() - _, _ = g.MultiExp(result, bases, scalars) - if !g.Equal(expected, result) { - t.Fatal("bad multi-exponentiation") +} + +func TestG2ClearCofactor(t *testing.T) { + g := NewG2() + for i := 0; i < fuz; i++ { + p0 := g.rand() + if g.InCorrectSubgroup(p0) { + t.Fatal("rand point should be out of correct subgroup") + } + g.ClearCofactor(p0) + if !g.InCorrectSubgroup(p0) { + t.Fatal("cofactor clearing is failed") + } } } @@ -224,24 +463,60 @@ func TestG2MapToCurve(t *testing.T) { expected []byte }{ { - u: make([]byte, 96), - expected: common.FromHex("0a67d12118b5a35bb02d2e86b3ebfa7e23410db93de39fb06d7025fa95e96ffa428a7a27c3ae4dd4b40bd251ac658892" + "018320896ec9eef9d5e619848dc29ce266f413d02dd31d9b9d44ec0c79cd61f18b075ddba6d7bd20b7ff27a4b324bfce" + "04c69777a43f0bda07679d5805e63f18cf4e0e7c6112ac7f70266d199b4f76ae27c6269a3ceebdae30806e9a76aadf5c" + "0260e03644d1a2c321256b3246bad2b895cad13890cbe6f85df55106a0d334604fb143c7a042d878006271865bc35941"), + u: make([]byte, 2*fpByteSize), + expected: fromHex(-1, "0a67d12118b5a35bb02d2e86b3ebfa7e23410db93de39fb06d7025fa95e96ffa428a7a27c3ae4dd4b40bd251ac658892", + "018320896ec9eef9d5e619848dc29ce266f413d02dd31d9b9d44ec0c79cd61f18b075ddba6d7bd20b7ff27a4b324bfce", + "04c69777a43f0bda07679d5805e63f18cf4e0e7c6112ac7f70266d199b4f76ae27c6269a3ceebdae30806e9a76aadf5c", + "0260e03644d1a2c321256b3246bad2b895cad13890cbe6f85df55106a0d334604fb143c7a042d878006271865bc35941", + ), }, { - u: common.FromHex("025fbc07711ba267b7e70c82caa70a16fbb1d470ae24ceef307f5e2000751677820b7013ad4e25492dcf30052d3e5eca" + "0e775d7827adf385b83e20e4445bd3fab21d7b4498426daf3c1d608b9d41e9edb5eda0df022e753b8bb4bc3bb7db4914"), - expected: common.FromHex("0d4333b77becbf9f9dfa3ca928002233d1ecc854b1447e5a71f751c9042d000f42db91c1d6649a5e0ad22bd7bf7398b8" + "027e4bfada0b47f9f07e04aec463c7371e68f2fd0c738cd517932ea3801a35acf09db018deda57387b0f270f7a219e4d" + "0cc76dc777ea0d447e02a41004f37a0a7b1fafb6746884e8d9fc276716ccf47e4e0899548a2ec71c2bdf1a2a50e876db" + "053674cba9ef516ddc218fedb37324e6c47de27f88ab7ef123b006127d738293c0277187f7e2f80a299a24d84ed03da7"), + u: fromHex(-1, + "025fbc07711ba267b7e70c82caa70a16fbb1d470ae24ceef307f5e2000751677820b7013ad4e25492dcf30052d3e5eca", + "0e775d7827adf385b83e20e4445bd3fab21d7b4498426daf3c1d608b9d41e9edb5eda0df022e753b8bb4bc3bb7db4914", + ), + expected: fromHex(-1, + "0d4333b77becbf9f9dfa3ca928002233d1ecc854b1447e5a71f751c9042d000f42db91c1d6649a5e0ad22bd7bf7398b8", + "027e4bfada0b47f9f07e04aec463c7371e68f2fd0c738cd517932ea3801a35acf09db018deda57387b0f270f7a219e4d", + "0cc76dc777ea0d447e02a41004f37a0a7b1fafb6746884e8d9fc276716ccf47e4e0899548a2ec71c2bdf1a2a50e876db", + "053674cba9ef516ddc218fedb37324e6c47de27f88ab7ef123b006127d738293c0277187f7e2f80a299a24d84ed03da7", + ), }, { - u: common.FromHex("1870a7dbfd2a1deb74015a3546b20f598041bf5d5202997956a94a368d30d3f70f18cdaa1d33ce970a4e16af961cbdcb" + "045ab31ce4b5a8ba7c4b2851b64f063a66cd1223d3c85005b78e1beee65e33c90ceef0244e45fc45a5e1d6eab6644fdb"), - expected: common.FromHex("18f0f87b40af67c056915dbaf48534c592524e82c1c2b50c3734d02c0172c80df780a60b5683759298a3303c5d942778" + "09349f1cb5b2e55489dcd45a38545343451cc30a1681c57acd4fb0a6db125f8352c09f4a67eb7d1d8242cb7d3405f97b" + "10a2ba341bc689ab947b7941ce6ef39be17acaab067bd32bd652b471ab0792c53a2bd03bdac47f96aaafe96e441f63c0" + "02f2d9deb2c7742512f5b8230bf0fd83ea42279d7d39779543c1a43b61c885982b611f6a7a24b514995e8a098496b811"), + u: fromHex(-1, + "1870a7dbfd2a1deb74015a3546b20f598041bf5d5202997956a94a368d30d3f70f18cdaa1d33ce970a4e16af961cbdcb", + "045ab31ce4b5a8ba7c4b2851b64f063a66cd1223d3c85005b78e1beee65e33c90ceef0244e45fc45a5e1d6eab6644fdb", + ), + expected: fromHex(-1, + "18f0f87b40af67c056915dbaf48534c592524e82c1c2b50c3734d02c0172c80df780a60b5683759298a3303c5d942778", + "09349f1cb5b2e55489dcd45a38545343451cc30a1681c57acd4fb0a6db125f8352c09f4a67eb7d1d8242cb7d3405f97b", + "10a2ba341bc689ab947b7941ce6ef39be17acaab067bd32bd652b471ab0792c53a2bd03bdac47f96aaafe96e441f63c0", + "02f2d9deb2c7742512f5b8230bf0fd83ea42279d7d39779543c1a43b61c885982b611f6a7a24b514995e8a098496b811", + ), }, { - u: common.FromHex("088fe329b054db8a6474f21a7fbfdf17b4c18044db299d9007af582c3d5f17d00e56d99921d4b5640fce44b05219b5de" + "0b6e6135a4cd31ba980ddbd115ac48abef7ec60e226f264d7befe002c165f3a496f36f76dd524efd75d17422558d10b4"), - expected: common.FromHex("19808ec5930a53c7cf5912ccce1cc33f1b3dcff24a53ce1cc4cba41fd6996dbed4843ccdd2eaf6a0cd801e562718d163" + "149fe43777d34f0d25430dea463889bd9393bdfb4932946db23671727081c629ebb98a89604f3433fba1c67d356a4af7" + "04783e391c30c83f805ca271e353582fdf19d159f6a4c39b73acbb637a9b8ac820cfbe2738d683368a7c07ad020e3e33" + "04c0d6793a766233b2982087b5f4a254f261003ccb3262ea7c50903eecef3e871d1502c293f9e063d7d293f6384f4551"), + u: fromHex(-1, + "088fe329b054db8a6474f21a7fbfdf17b4c18044db299d9007af582c3d5f17d00e56d99921d4b5640fce44b05219b5de", + "0b6e6135a4cd31ba980ddbd115ac48abef7ec60e226f264d7befe002c165f3a496f36f76dd524efd75d17422558d10b4", + ), + expected: fromHex(-1, + "19808ec5930a53c7cf5912ccce1cc33f1b3dcff24a53ce1cc4cba41fd6996dbed4843ccdd2eaf6a0cd801e562718d163", + "149fe43777d34f0d25430dea463889bd9393bdfb4932946db23671727081c629ebb98a89604f3433fba1c67d356a4af7", + "04783e391c30c83f805ca271e353582fdf19d159f6a4c39b73acbb637a9b8ac820cfbe2738d683368a7c07ad020e3e33", + "04c0d6793a766233b2982087b5f4a254f261003ccb3262ea7c50903eecef3e871d1502c293f9e063d7d293f6384f4551", + ), }, { - u: common.FromHex("03df16a66a05e4c1188c234788f43896e0565bfb64ac49b9639e6b284cc47dad73c47bb4ea7e677db8d496beb907fbb6" + "0f45b50647d67485295aa9eb2d91a877b44813677c67c8d35b2173ff3ba95f7bd0806f9ca8a1436b8b9d14ee81da4d7e"), - expected: common.FromHex("0b8e0094c886487870372eb6264613a6a087c7eb9804fab789be4e47a57b29eb19b1983a51165a1b5eb025865e9fc63a" + "0804152cbf8474669ad7d1796ab92d7ca21f32d8bed70898a748ed4e4e0ec557069003732fc86866d938538a2ae95552" + "14c80f068ece15a3936bb00c3c883966f75b4e8d9ddde809c11f781ab92d23a2d1d103ad48f6f3bb158bf3e3a4063449" + "09e5c8242dd7281ad32c03fe4af3f19167770016255fb25ad9b67ec51d62fade31a1af101e8f6172ec2ee8857662be3a"), + u: fromHex(-1, + "03df16a66a05e4c1188c234788f43896e0565bfb64ac49b9639e6b284cc47dad73c47bb4ea7e677db8d496beb907fbb6", + "0f45b50647d67485295aa9eb2d91a877b44813677c67c8d35b2173ff3ba95f7bd0806f9ca8a1436b8b9d14ee81da4d7e", + ), + expected: fromHex(-1, + "0b8e0094c886487870372eb6264613a6a087c7eb9804fab789be4e47a57b29eb19b1983a51165a1b5eb025865e9fc63a", + "0804152cbf8474669ad7d1796ab92d7ca21f32d8bed70898a748ed4e4e0ec557069003732fc86866d938538a2ae95552", + "14c80f068ece15a3936bb00c3c883966f75b4e8d9ddde809c11f781ab92d23a2d1d103ad48f6f3bb158bf3e3a4063449", + "09e5c8242dd7281ad32c03fe4af3f19167770016255fb25ad9b67ec51d62fade31a1af101e8f6172ec2ee8857662be3a", + ), }, } { g := NewG2() @@ -255,6 +530,114 @@ func TestG2MapToCurve(t *testing.T) { } } +func TestG2EncodeToCurve(t *testing.T) { + domain := []byte("BLS12381G2_XMD:SHA-256_SSWU_NU_TESTGEN") + for i, v := range []struct { + msg []byte + expected []byte + }{ + { + msg: []byte(""), + expected: fromHex(-1, + "0d4333b77becbf9f9dfa3ca928002233d1ecc854b1447e5a71f751c9042d000f42db91c1d6649a5e0ad22bd7bf7398b8", + "027e4bfada0b47f9f07e04aec463c7371e68f2fd0c738cd517932ea3801a35acf09db018deda57387b0f270f7a219e4d", + "0cc76dc777ea0d447e02a41004f37a0a7b1fafb6746884e8d9fc276716ccf47e4e0899548a2ec71c2bdf1a2a50e876db", + "053674cba9ef516ddc218fedb37324e6c47de27f88ab7ef123b006127d738293c0277187f7e2f80a299a24d84ed03da7", + ), + }, + { + msg: []byte("abc"), + expected: fromHex(-1, + "18f0f87b40af67c056915dbaf48534c592524e82c1c2b50c3734d02c0172c80df780a60b5683759298a3303c5d942778", + "09349f1cb5b2e55489dcd45a38545343451cc30a1681c57acd4fb0a6db125f8352c09f4a67eb7d1d8242cb7d3405f97b", + "10a2ba341bc689ab947b7941ce6ef39be17acaab067bd32bd652b471ab0792c53a2bd03bdac47f96aaafe96e441f63c0", + "02f2d9deb2c7742512f5b8230bf0fd83ea42279d7d39779543c1a43b61c885982b611f6a7a24b514995e8a098496b811", + ), + }, + { + msg: []byte("abcdef0123456789"), + expected: fromHex(-1, + "19808ec5930a53c7cf5912ccce1cc33f1b3dcff24a53ce1cc4cba41fd6996dbed4843ccdd2eaf6a0cd801e562718d163", + "149fe43777d34f0d25430dea463889bd9393bdfb4932946db23671727081c629ebb98a89604f3433fba1c67d356a4af7", + "04783e391c30c83f805ca271e353582fdf19d159f6a4c39b73acbb637a9b8ac820cfbe2738d683368a7c07ad020e3e33", + "04c0d6793a766233b2982087b5f4a254f261003ccb3262ea7c50903eecef3e871d1502c293f9e063d7d293f6384f4551", + ), + }, + { + msg: []byte("a512_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + expected: fromHex(-1, + "0b8e0094c886487870372eb6264613a6a087c7eb9804fab789be4e47a57b29eb19b1983a51165a1b5eb025865e9fc63a", + "0804152cbf8474669ad7d1796ab92d7ca21f32d8bed70898a748ed4e4e0ec557069003732fc86866d938538a2ae95552", + "14c80f068ece15a3936bb00c3c883966f75b4e8d9ddde809c11f781ab92d23a2d1d103ad48f6f3bb158bf3e3a4063449", + "09e5c8242dd7281ad32c03fe4af3f19167770016255fb25ad9b67ec51d62fade31a1af101e8f6172ec2ee8857662be3a", + ), + }, + } { + g := NewG2() + p0, err := g.EncodeToCurve(v.msg, domain) + if err != nil { + t.Fatal("encode to point fails", i, err) + } + if !bytes.Equal(g.ToBytes(p0), v.expected) { + t.Fatal("encode to point fails x", i) + } + } +} + +func TestG2HashToCurve(t *testing.T) { + domain := []byte("BLS12381G2_XMD:SHA-256_SSWU_RO_TESTGEN") + for i, v := range []struct { + msg []byte + expected []byte + }{ + { + msg: []byte(""), + expected: fromHex(-1, + "0fbdae26f9f9586a46d4b0b70390d09064ef2afe5c99348438a3c7d9756471e015cb534204c1b6824617a85024c772dc", + "0a650bd36ae7455cb3fe5d8bb1310594551456f5c6593aec9ee0c03d2f6cb693bd2c5e99d4e23cbaec767609314f51d3", + "02e5cf8f9b7348428cc9e66b9a9b36fe45ba0b0a146290c3a68d92895b1af0e1f2d9f889fb412670ae8478d8abd4c5aa", + "0d8d49e7737d8f9fc5cef7c4b8817633103faf2613016cb86a1f3fc29968fe2413e232d9208d2d74a89bf7a48ac36f83", + ), + }, + { + msg: []byte("abc"), + expected: fromHex(-1, + "03578447618463deb106b60e609c6f7cc446dc6035f84a72801ba17c94cd800583b493b948eff0033f09086fdd7f6175", + "1953ce6d4267939c7360756d9cca8eb34aac4633ef35369a7dc249445069888e7d1b3f9d2e75fbd468fbcbba7110ea02", + "0184d26779ae9d4670aca9b267dbd4d3b30443ad05b8546d36a195686e1ccc3a59194aea05ed5bce7c3144a29ec047c4", + "0882ab045b8fe4d7d557ebb59a63a35ac9f3d312581b509af0f8eaa2960cbc5e1e36bb969b6e22980b5cbdd0787fcf4e", + ), + }, + { + msg: []byte("abcdef0123456789"), + expected: fromHex(-1, + "195fad48982e186ce3c5c82133aefc9b26d55979b6f530992a8849d4263ec5d57f7a181553c8799bcc83da44847bdc8d", + "17b461fc3b96a30c2408958cbfa5f5927b6063a8ad199d5ebf2d7cdeffa9c20c85487204804fab53f950b2f87db365aa", + "005cdf3d984e3391e7e969276fb4bc02323c5924a4449af167030d855acc2600cf3d4fab025432c6d868c79571a95bef", + "174a3473a3af2d0302b9065e895ca4adba4ece6ce0b41148ba597001abb152f852dd9a96fb45c9de0a43d944746f833e", + ), + }, + { + msg: []byte("a512_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + expected: fromHex(-1, + "123b6bd9feeba26dd4ad00f8bfda2718c9700dc093ea5287d7711844644eb981848316d3f3f57d5d3a652c6cdc816aca", + "0a162306f3b0f2bb326f0c4fb0e1fea020019c3af796dcd1d7264f50ddae94cacf3cade74603834d44b9ab3d5d0a6c98", + "05483f3b96d9252dd4fc0868344dfaf3c9d145e3387db23fa8e449304fab6a7b6ec9c15f05c0a1ea66ff0efcc03e001a", + "15c1d4f1a685bb63ee67ca1fd96155e3d091e852a684b78d085fd34f6091e5249ddddbdcf2e7ec82ce6c04c63647eeb7", + ), + }, + } { + g := NewG2() + p0, err := g.HashToCurve(v.msg, domain) + if err != nil { + t.Fatal("encode to point fails", i, err) + } + if !bytes.Equal(g.ToBytes(p0), v.expected) { + t.Fatal("encode to point fails x", i) + } + } +} + func BenchmarkG2Add(t *testing.B) { g2 := NewG2() a, b, c := g2.rand(), g2.rand(), PointG2{} @@ -264,18 +647,112 @@ func BenchmarkG2Add(t *testing.B) { } } -func BenchmarkG2Mul(t *testing.B) { - worstCaseScalar, _ := new(big.Int).SetString("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16) +func BenchmarkG2MulWNAF(t *testing.B) { + g := NewG2() + p := new(PointG2).Set(&g2One) + s, _ := new(Fr).Rand(rand.Reader) + sBig := s.ToBig() + res := new(PointG2) + t.Run("Naive", func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.mulScalar(res, p, s) + } + }) + for i := 1; i < 8; i++ { + wnafMulWindowG2 = uint(i) + t.Run(fmt.Sprintf("Fr, window: %d", i), func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.wnafMulFr(res, p, s) + } + }) + t.Run(fmt.Sprintf("Big, window: %d", i), func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.wnafMulBig(res, p, sBig) + } + }) + } +} + +func BenchmarkG2MulGLV(t *testing.B) { + + g := NewG2() + p := new(PointG2).Set(&g2One) + s, _ := new(Fr).Rand(rand.Reader) + sBig := s.ToBig() + res := new(PointG2) + t.Run("Naive", func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.mulScalar(res, p, s) + } + }) + for i := 1; i < 8; i++ { + glvMulWindowG2 = uint(i) + t.Run(fmt.Sprintf("Fr, window: %d", i), func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.glvMulFr(res, p, s) + } + }) + t.Run(fmt.Sprintf("Big, window: %d", i), func(t *testing.B) { + t.ResetTimer() + for i := 0; i < t.N; i++ { + g.glvMulBig(res, p, sBig) + } + }) + } +} + +func BenchmarkG2MultiExp(t *testing.B) { + g := NewG2() + v := func(n int) ([]*PointG2, []*Fr) { + bases := make([]*PointG2, n) + scalars := make([]*Fr, n) + var err error + for i := 0; i < n; i++ { + scalars[i], err = new(Fr).Rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + bases[i] = g.randAffine() + } + return bases, scalars + } + for _, i := range []int{2, 10, 100, 1000} { + t.Run(fmt.Sprint(i), func(t *testing.B) { + bases, scalars := v(i) + result := g.New() + t.ResetTimer() + for i := 0; i < t.N; i++ { + _, _ = g.MultiExp(result, bases, scalars) + } + }) + } +} + +func BenchmarkG2ClearCofactor(t *testing.B) { g2 := NewG2() - a, e, c := g2.rand(), worstCaseScalar, PointG2{} + a := g2.rand() t.ResetTimer() for i := 0; i < t.N; i++ { - g2.MulScalar(&c, a, e) + g2.ClearCofactor(a) + } +} + +func BenchmarkG2SubgroupCheck(t *testing.B) { + g2 := NewG2() + a := g2.rand() + t.ResetTimer() + for i := 0; i < t.N; i++ { + g2.InCorrectSubgroup(a) } } func BenchmarkG2SWUMap(t *testing.B) { - a := make([]byte, 96) + a := fromHex(2*fpByteSize, "0x1234") g2 := NewG2() t.ResetTimer() for i := 0; i < t.N; i++ { diff --git a/crypto/bls12381/glv.go b/crypto/bls12381/glv.go new file mode 100644 index 0000000000..0e61818928 --- /dev/null +++ b/crypto/bls12381/glv.go @@ -0,0 +1,200 @@ +package bls12381 + +import ( + "math/big" +) + +// Guide to Pairing Based Cryptography +// 6.3.2. Decompositions for the k = 12 BLS Family + +// glvQ1 = x^2 * R / q +var glvQ1 = &Fr{0x63f6e522f6cfee30, 0x7c6becf1e01faadd, 0x1, 0} +var glvQ1Big = bigFromHex("0x017c6becf1e01faadd63f6e522f6cfee30") + +// glvQ2 = R / q = 2 +var glvQ2 = &Fr{0x02, 0, 0, 0} +var glvQ2Big = bigFromHex("0x02") + +// glvB1 = x^2 - 1 = 0xac45a4010001a40200000000ffffffff +var glvB1 = &Fr{0x00000000ffffffff, 0xac45a4010001a402, 0, 0} +var glvB1Big = bigFromHex("0xac45a4010001a40200000000ffffffff") + +// glvB2 = x^2 = 0xac45a4010001a4020000000100000000 +var glvB2 = &Fr{0x0000000100000000, 0xac45a4010001a402, 0, 0} +var glvB2Big = bigFromHex("0xac45a4010001a4020000000100000000") + +// glvLambdaA = x^2 - 1 +var glvLambda = &Fr{0x00000000ffffffff, 0xac45a4010001a402, 0, 0} +var glvLambdaBig = bigFromHex("0xac45a4010001a40200000000ffffffff") + +// halfR = 2**256 / 2 +var halfR = &wideFr{0, 0, 0, 0x8000000000000000, 0, 0, 0} +var halfRBig = bigFromHex("0x8000000000000000000000000000000000000000000000000000000000000000") + +// r128 = 2**128 - 1 +var r128 = &Fr{0xffffffffffffffff, 0xffffffffffffffff, 0, 0} + +// glvPhi1 ^ 3 = 1 +var glvPhi1 = &fe{0xcd03c9e48671f071, 0x5dab22461fcda5d2, 0x587042afd3851b95, 0x8eb60ebe01bacb9e, 0x03f97d6e83d050d2, 0x18f0206554638741} + +// glvPhi2 ^ 3 = 1 +var glvPhi2 = &fe{0x30f1361b798a64e8, 0xf3b8ddab7ece5a2a, 0x16a8ca3ac61577f7, 0xc26a2ff874fd029b, 0x3636b76660701c6e, 0x051ba4ab241b6160} + +var glvMulWindowG1 uint = 4 +var glvMulWindowG2 uint = 4 + +type glvVector interface { + wnaf(w uint) (nafNumber, nafNumber) +} + +type glvVectorFr struct { + k1 *Fr + k2 *Fr + neg1 bool + neg2 bool +} + +type glvVectorBig struct { + k1 *big.Int + k2 *big.Int +} + +func (v *glvVectorFr) wnaf(w uint) (nafNumber, nafNumber) { + naf1 := v.k1.toWNAF(w) + naf2 := v.k2.toWNAF(w) + if v.neg1 { + naf1.neg() + } + if !v.neg2 { + naf2.neg() + } + return naf1, naf2 +} + +func (v *glvVectorBig) wnaf(w uint) (nafNumber, nafNumber) { + naf1, naf2 := bigToWNAF(v.k1, w), bigToWNAF(v.k2, w) + zero := new(big.Int) + if v.k1.Cmp(zero) < 0 { + naf1.neg() + } + if v.k2.Cmp(zero) > 0 { + naf2.neg() + } + return naf1, naf2 +} + +func (v *glvVectorFr) new(m *Fr) *glvVectorFr { + // Guide to Pairing Based Cryptography + // 6.3.2. Decompositions for the k = 12 BLS Family + + // alpha1 = round(x^2 * m / r) + alpha1 := alpha1(m) + // alpha2 = round(m / r) + alpha2 := alpha2(m) + + z1, z2 := new(Fr), new(Fr) + + // z1 = (x^2 - 1) * round(x^2 * m / r) + z1.Mul(alpha1, glvB1) + // z2 = x^2 * round(m / r) + z2.Mul(alpha2, glvB2) + + k1, k2 := new(Fr), new(Fr) + // k1 = m - z1 - alpha2 + k1.Sub(m, z1) + k1.Sub(k1, alpha2) + + // k2 = z2 - alpha1 + k2.Sub(z2, alpha1) + + if k1.Cmp(r128) == 1 { + k1.Neg(k1) + v.neg1 = true + } + v.k1 = new(Fr).Set(k1) + if k2.Cmp(r128) == 1 { + k2.Neg(k2) + v.neg2 = true + } + v.k2 = new(Fr).Set(k2) + return v +} + +func (v *glvVectorBig) new(m *big.Int) *glvVectorBig { + // Guide to Pairing Based Cryptography + // 6.3.2. Decompositions for the k = 12 BLS Family + + // alpha1 = round(x^2 * m / r) + alpha1 := new(big.Int).Mul(m, glvQ1Big) + alpha1.Add(alpha1, halfRBig) + alpha1.Rsh(alpha1, fourWordBitSize) + + // alpha2 = round(m / r) + alpha2 := new(big.Int).Mul(m, glvQ2Big) + alpha2.Add(alpha2, halfRBig) + alpha2.Rsh(alpha2, fourWordBitSize) + + z1, z2 := new(big.Int), new(big.Int) + // z1 = (x^2 - 1) * round(x^2 * m / r) + z1.Mul(alpha1, glvB1Big).Mod(z1, qBig) + // z2 = x^2 * round(m / r) + z2.Mul(alpha2, glvB2Big).Mod(z2, qBig) + + k1, k2 := new(big.Int), new(big.Int) + + // k1 = m - z1 - alpha2 + k1.Sub(m, z1) + k1.Sub(k1, alpha2) + + // k2 = z2 - alpha1 + k2.Sub(z2, alpha1) + + v.k1 = new(big.Int).Set(k1) + v.k2 = new(big.Int).Set(k2) + return v +} + +// round(x^2 * m / q) +func alpha1(m *Fr) *Fr { + a := new(wideFr) + a.mul(m, glvQ1) + return a.round() +} + +// round(m / q) +func alpha2(m *Fr) *Fr { + a := new(wideFr) + a.mul(m, glvQ2) + return a.round() +} + +func phi(a, b *fe) { + mul(a, b, glvPhi1) +} + +func (e *fp2) phi(a, b *fe2) { + mul(&a[0], &b[0], glvPhi2) + mul(&a[1], &b[1], glvPhi2) +} + +func (g *G1) glvEndomorphism(r, p *PointG1) { + t := g.Affine(p) + if g.IsZero(p) { + r.Zero() + return + } + r[1].set(&t[1]) + phi(&r[0], &t[0]) + r[2].one() +} + +func (g *G2) glvEndomorphism(r, p *PointG2) { + t := g.Affine(p) + if g.IsZero(p) { + r.Zero() + return + } + r[1].set(&t[1]) + g.f.phi(&r[0], &t[0]) + r[2].one() +} diff --git a/crypto/bls12381/glv_test.go b/crypto/bls12381/glv_test.go new file mode 100644 index 0000000000..2c646068de --- /dev/null +++ b/crypto/bls12381/glv_test.go @@ -0,0 +1,134 @@ +package bls12381 + +import ( + "crypto/rand" + "math/big" + "testing" +) + +func TestGLVConstruction(t *testing.T) { + t.Run("Parameters", func(t *testing.T) { + t0, t1 := new(Fr), new(Fr) + one := new(Fr).setUint64(1) + t0.Square(glvLambda) + t0.Add(t0, glvLambda) + t1.Sub(&q, one) + if !t0.Equal(t1) { + t.Fatal("lambda1^2 + lambda1 + 1 = 0") + } + c0 := new(fe) + square(c0, glvPhi1) + mul(c0, c0, glvPhi1) + if !c0.isOne() { + t.Fatal("phi1^3 = 1") + } + square(c0, glvPhi2) + mul(c0, c0, glvPhi2) + if !c0.isOne() { + t.Fatal("phi2^3 = 1") + } + }) + t.Run("Endomorphism G1", func(t *testing.T) { + g := NewG1() + { + p0, p1 := g.randAffine(), g.New() + g.MulScalar(p1, p0, glvLambda) + g.Affine(p1) + r := g.New() + g.glvEndomorphism(r, p0) + if !g.Equal(r, p1) { + t.Fatal("f(x, y) = (phi * x, y)") + } + } + }) + t.Run("Endomorphism G2", func(t *testing.T) { + g := NewG2() + { + p0, p1 := g.randAffine(), g.New() + g.MulScalar(p1, p0, glvLambda) + g.Affine(p1) + r := g.New() + g.glvEndomorphism(r, p0) + if !g.Equal(r, p1) { + t.Fatal("f(x, y) = (phi * x, y)") + } + } + }) + t.Run("Scalar Decomposition", func(t *testing.T) { + for i := 0; i < fuz; i++ { + m, err := new(Fr).Rand(rand.Reader) + if err != nil { + t.Fatal(err) + } + mBig := m.ToBig() + var vFr *glvVectorFr + var vBig *glvVectorBig + { + vFr = new(glvVectorFr).new(m) + v := vFr + + if v.k1.Cmp(r128) >= 0 { + t.Fatal("bad scalar component, k1") + } + if v.k2.Cmp(r128) >= 0 { + t.Fatal("bad scalar component, k2") + } + + k := new(Fr) + if v.neg1 && v.neg2 { + k.Mul(glvLambda, v.k2) + k.Sub(k, v.k1) + } else if v.neg1 { + k.Mul(glvLambda, v.k2) + k.Add(k, v.k1) + k.Neg(k) + } else if v.neg2 { + k.Mul(glvLambda, v.k2) + k.Add(v.k1, k) + } else { + k.Mul(glvLambda, v.k2) + k.Sub(v.k1, k) + } + + if !k.Equal(m) { + t.Fatal("scalar decomposing failed") + } + } + + r128Big := r128.ToBig() + { + vBig = new(glvVectorBig).new(mBig) + + if new(big.Int).Abs(vBig.k1).Cmp(r128Big) >= 0 { + t.Fatal("bad scalar component, big k1") + } + if new(big.Int).Abs(vBig.k2).Cmp(r128Big) >= 0 { + t.Fatal("bad scalar component, big k2") + } + + k := new(big.Int) + k.Mul(glvLambdaBig, vBig.k2) + k.Sub(vBig.k1, k).Mod(k, qBig) + if k.Cmp(mBig) != 0 { + t.Fatal("scalar decomposing with big.Int failed", i) + } + } + + zeroBig := new(big.Int) + k1Abs, k2Abs := new(big.Int).Abs(vBig.k1), new(big.Int).Abs(vBig.k2) + + if vFr.neg1 != (vBig.k1.Cmp(zeroBig) == -1) { + t.Fatal("cross: scalar decomposing with failed neg1") + } + if vFr.neg2 != (vBig.k2.Cmp(zeroBig) == -1) { + t.Fatal("cross: scalar decomposing with failed neg2") + } + if k1Abs.Cmp(vFr.k1.ToBig()) != 0 { + t.Fatal("cross: scalar decomposing with failed k1", i) + } + if k2Abs.Cmp(vFr.k2.ToBig()) != 0 { + t.Fatal("cross: scalar decomposing with failed k2", i) + } + } + }) +} diff --git a/crypto/bls12381/gt.go b/crypto/bls12381/gt.go index 2ac265e956..3e28bb366f 100644 --- a/crypto/bls12381/gt.go +++ b/crypto/bls12381/gt.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( @@ -29,6 +13,7 @@ type GT struct { fp12 *fp12 } +// Set copies given value into the destination func (e *E) Set(e2 *E) *E { return e.set(e2) } @@ -57,7 +42,7 @@ func NewGT() *GT { // Q returns group order in big.Int. func (g *GT) Q() *big.Int { - return new(big.Int).Set(q) + return new(big.Int).Set(qBig) } // FromBytes expects 576 byte input and returns target group element @@ -81,7 +66,7 @@ func (g *GT) ToBytes(e *E) []byte { // IsValid checks whether given target group element is in correct subgroup. func (g *GT) IsValid(e *E) bool { r := g.New() - g.fp12.exp(r, e, q) + g.fp12.exp(r, e, qBig) return r.isOne() } @@ -92,12 +77,12 @@ func (g *GT) New() *E { // Add adds two field element `a` and `b` and assigns the result to the element in first argument. func (g *GT) Add(c, a, b *E) { - g.fp12.add(c, a, b) + fp12Add(c, a, b) } // Sub subtracts two field element `a` and `b`, and assigns the result to the element in first argument. func (g *GT) Sub(c, a, b *E) { - g.fp12.sub(c, a, b) + fp12Sub(c, a, b) } // Mul multiplies two field element `a` and `b` and assigns the result to the element in first argument. @@ -107,7 +92,8 @@ func (g *GT) Mul(c, a, b *E) { // Square squares an element `a` and assigns the result to the element in first argument. func (g *GT) Square(c, a *E) { - g.fp12.cyclotomicSquare(c, a) + c.set(a) + g.fp12.cyclotomicSquare(c) } // Exp exponents an element `a` by a scalar `s` and assigns the result to the element in first argument. diff --git a/crypto/bls12381/hash_to_field.go b/crypto/bls12381/hash_to_field.go new file mode 100644 index 0000000000..9e44007691 --- /dev/null +++ b/crypto/bls12381/hash_to_field.go @@ -0,0 +1,70 @@ +package bls12381 + +import ( + "crypto/sha256" + "errors" +) + +func hashToFpXMDSHA256(msg []byte, domain []byte, count int) ([]*fe, error) { + randBytes, err := expandMsgSHA256XMD(msg, domain, count*64) + if err != nil { + return nil, err + } + els := make([]*fe, count) + for i := 0; i < count; i++ { + els[i], err = from64Bytes(randBytes[i*64 : (i+1)*64]) + if err != nil { + return nil, err + } + } + return els, nil +} + +func expandMsgSHA256XMD(msg []byte, domain []byte, outLen int) ([]byte, error) { + h := sha256.New() + domainLen := uint8(len(domain)) + if domainLen > 255 { + return nil, errors.New("invalid domain length") + } + // DST_prime = DST || I2OSP(len(DST), 1) + // b_0 = H(Z_pad || msg || l_i_b_str || I2OSP(0, 1) || DST_prime) + _, _ = h.Write(make([]byte, h.BlockSize())) + _, _ = h.Write(msg) + _, _ = h.Write([]byte{uint8(outLen >> 8), uint8(outLen)}) + _, _ = h.Write([]byte{0}) + _, _ = h.Write(domain) + _, _ = h.Write([]byte{domainLen}) + b0 := h.Sum(nil) + + // b_1 = H(b_0 || I2OSP(1, 1) || DST_prime) + h.Reset() + _, _ = h.Write(b0) + _, _ = h.Write([]byte{1}) + _, _ = h.Write(domain) + _, _ = h.Write([]byte{domainLen}) + b1 := h.Sum(nil) + + // b_i = H(strxor(b_0, b_(i - 1)) || I2OSP(i, 1) || DST_prime) + ell := (outLen + h.Size() - 1) / h.Size() + bi := b1 + out := make([]byte, outLen) + for i := 1; i < ell; i++ { + h.Reset() + // b_i = H(strxor(b_0, b_(i - 1)) || I2OSP(i, 1) || DST_prime) + tmp := make([]byte, h.Size()) + for j := 0; j < h.Size(); j++ { + tmp[j] = b0[j] ^ bi[j] + } + _, _ = h.Write(tmp) + _, _ = h.Write([]byte{1 + uint8(i)}) + _, _ = h.Write(domain) + _, _ = h.Write([]byte{domainLen}) + + // b_1 || ... || b_(ell - 1) + copy(out[(i-1)*h.Size():i*h.Size()], bi[:]) + bi = h.Sum(nil) + } + // b_ell + copy(out[(ell-1)*h.Size():], bi[:]) + return out[:outLen], nil +} diff --git a/crypto/bls12381/isogeny.go b/crypto/bls12381/isogeny.go index a63f585dd0..80274ef05c 100644 --- a/crypto/bls12381/isogeny.go +++ b/crypto/bls12381/isogeny.go @@ -1,82 +1,70 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 // isogenyMapG1 applies 11-isogeny map for BLS12-381 G1 defined at draft-irtf-cfrg-hash-to-curve-06. func isogenyMapG1(x, y *fe) { - // https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06#appendix-C.2 - params := isogenyConstantsG1 - degree := 15 xNum, xDen, yNum, yDen := new(fe), new(fe), new(fe), new(fe) - xNum.set(params[0][degree]) - xDen.set(params[1][degree]) - yNum.set(params[2][degree]) - yDen.set(params[3][degree]) - for i := degree - 1; i >= 0; i-- { + xNum.set(isogenyConstansG1[0][15]) + xDen.set(isogenyConstansG1[1][15]) + yNum.set(isogenyConstansG1[2][15]) + yDen.set(isogenyConstansG1[3][15]) + for i := 14; i > -1; i-- { mul(xNum, xNum, x) mul(xDen, xDen, x) mul(yNum, yNum, x) mul(yDen, yDen, x) - add(xNum, xNum, params[0][i]) - add(xDen, xDen, params[1][i]) - add(yNum, yNum, params[2][i]) - add(yDen, yDen, params[3][i]) + addAssign(xNum, isogenyConstansG1[0][i]) + addAssign(xDen, isogenyConstansG1[1][i]) + addAssign(yNum, isogenyConstansG1[2][i]) + addAssign(yDen, isogenyConstansG1[3][i]) } inverse(xDen, xDen) inverse(yDen, yDen) - mul(xNum, xNum, xDen) + mul(x, xNum, xDen) mul(yNum, yNum, yDen) - mul(yNum, yNum, y) - x.set(xNum) - y.set(yNum) + mul(y, y, yNum) } -// isogenyMapG2 applies 11-isogeny map for BLS12-381 G1 defined at draft-irtf-cfrg-hash-to-curve-06. +// isogenyMapG2 applies 3-isogeny map for BLS12-381 G2 defined at draft-irtf-cfrg-hash-to-curve-06. func isogenyMapG2(e *fp2, x, y *fe2) { if e == nil { e = newFp2() } - // https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06#appendix-C.2 - params := isogenyConstantsG2 - degree := 3 - xNum := new(fe2).set(params[0][degree]) - xDen := new(fe2).set(params[1][degree]) - yNum := new(fe2).set(params[2][degree]) - yDen := new(fe2).set(params[3][degree]) - for i := degree - 1; i >= 0; i-- { - e.mul(xNum, xNum, x) - e.mul(xDen, xDen, x) - e.mul(yNum, yNum, x) - e.mul(yDen, yDen, x) - e.add(xNum, xNum, params[0][i]) - e.add(xDen, xDen, params[1][i]) - e.add(yNum, yNum, params[2][i]) - e.add(yDen, yDen, params[3][i]) - } + xNum := new(fe2).set(isogenyConstantsG2[0][3]) + xDen := new(fe2).set(x) + yNum := new(fe2).set(isogenyConstantsG2[2][3]) + yDen := new(fe2).set(x) + + e.mulAssign(xNum, x) + e.mulAssign(yNum, x) + fp2AddAssign(xNum, isogenyConstantsG2[0][2]) + fp2AddAssign(yNum, isogenyConstantsG2[2][2]) + fp2AddAssign(yDen, isogenyConstantsG2[3][2]) + + e.mulAssign(xNum, x) + e.mulAssign(yNum, x) + e.mulAssign(yDen, x) + fp2AddAssign(xNum, isogenyConstantsG2[0][1]) + fp2AddAssign(xDen, isogenyConstantsG2[1][1]) + fp2AddAssign(yNum, isogenyConstantsG2[2][1]) + fp2AddAssign(yDen, isogenyConstantsG2[3][1]) + + e.mulAssign(xNum, x) + e.mulAssign(xDen, x) + e.mulAssign(yNum, x) + e.mulAssign(yDen, x) + fp2AddAssign(xNum, isogenyConstantsG2[0][0]) + fp2AddAssign(xDen, isogenyConstantsG2[1][0]) + fp2AddAssign(yNum, isogenyConstantsG2[2][0]) + fp2AddAssign(yDen, isogenyConstantsG2[3][0]) + e.inverse(xDen, xDen) e.inverse(yDen, yDen) - e.mul(xNum, xNum, xDen) - e.mul(yNum, yNum, yDen) - e.mul(yNum, yNum, y) - x.set(xNum) - y.set(yNum) + e.mul(x, xNum, xDen) + e.mulAssign(yNum, yDen) + e.mulAssign(y, yNum) } -var isogenyConstantsG1 = [4][16]*fe{ +var isogenyConstansG1 = [4][16]*fe{ { {0x4d18b6f3af00131c, 0x19fa219793fee28c, 0x3f2885f1467f19ae, 0x23dcea34f2ffb304, 0xd15b58d2ffc00054, 0x0913be200a20bef4}, {0x898985385cdbbd8b, 0x3c79e43cc7d966aa, 0x1597e193f4cd233a, 0x8637ef1e4d6623ad, 0x11b22deed20d827b, 0x07097bc5998784ad}, @@ -154,74 +142,74 @@ var isogenyConstantsG1 = [4][16]*fe{ var isogenyConstantsG2 = [4][4]*fe2{ { { - fe{0x47f671c71ce05e62, 0x06dd57071206393e, 0x7c80cd2af3fd71a2, 0x048103ea9e6cd062, 0xc54516acc8d037f6, 0x13808f550920ea41}, - fe{0x47f671c71ce05e62, 0x06dd57071206393e, 0x7c80cd2af3fd71a2, 0x048103ea9e6cd062, 0xc54516acc8d037f6, 0x13808f550920ea41}, + {0x47f671c71ce05e62, 0x06dd57071206393e, 0x7c80cd2af3fd71a2, 0x048103ea9e6cd062, 0xc54516acc8d037f6, 0x13808f550920ea41}, + {0x47f671c71ce05e62, 0x06dd57071206393e, 0x7c80cd2af3fd71a2, 0x048103ea9e6cd062, 0xc54516acc8d037f6, 0x13808f550920ea41}, }, { - fe{0, 0, 0, 0, 0, 0}, - fe{0x5fe55555554c71d0, 0x873fffdd236aaaa3, 0x6a6b4619b26ef918, 0x21c2888408874945, 0x2836cda7028cabc5, 0x0ac73310a7fd5abd}, + {0, 0, 0, 0, 0, 0}, + {0x5fe55555554c71d0, 0x873fffdd236aaaa3, 0x6a6b4619b26ef918, 0x21c2888408874945, 0x2836cda7028cabc5, 0x0ac73310a7fd5abd}, }, { - fe{0x0a0c5555555971c3, 0xdb0c00101f9eaaae, 0xb1fb2f941d797997, 0xd3960742ef416e1c, 0xb70040e2c20556f4, 0x149d7861e581393b}, - fe{0xaff2aaaaaaa638e8, 0x439fffee91b55551, 0xb535a30cd9377c8c, 0x90e144420443a4a2, 0x941b66d3814655e2, 0x0563998853fead5e}, + {0x0a0c5555555971c3, 0xdb0c00101f9eaaae, 0xb1fb2f941d797997, 0xd3960742ef416e1c, 0xb70040e2c20556f4, 0x149d7861e581393b}, + {0xaff2aaaaaaa638e8, 0x439fffee91b55551, 0xb535a30cd9377c8c, 0x90e144420443a4a2, 0x941b66d3814655e2, 0x0563998853fead5e}, }, { - fe{0x40aac71c71c725ed, 0x190955557a84e38e, 0xd817050a8f41abc3, 0xd86485d4c87f6fb1, 0x696eb479f885d059, 0x198e1a74328002d2}, - fe{0, 0, 0, 0, 0, 0}, + {0x40aac71c71c725ed, 0x190955557a84e38e, 0xd817050a8f41abc3, 0xd86485d4c87f6fb1, 0x696eb479f885d059, 0x198e1a74328002d2}, + {0, 0, 0, 0, 0, 0}, }, }, { { - fe{0, 0, 0, 0, 0, 0}, - fe{0x1f3affffff13ab97, 0xf25bfc611da3ff3e, 0xca3757cb3819b208, 0x3e6427366f8cec18, 0x03977bc86095b089, 0x04f69db13f39a952}, + {0, 0, 0, 0, 0, 0}, + {0x1f3affffff13ab97, 0xf25bfc611da3ff3e, 0xca3757cb3819b208, 0x3e6427366f8cec18, 0x03977bc86095b089, 0x04f69db13f39a952}, }, { - fe{0x447600000027552e, 0xdcb8009a43480020, 0x6f7ee9ce4a6e8b59, 0xb10330b7c0a95bc6, 0x6140b1fcfb1e54b7, 0x0381be097f0bb4e1}, - fe{0x7588ffffffd8557d, 0x41f3ff646e0bffdf, 0xf7b1e8d2ac426aca, 0xb3741acd32dbb6f8, 0xe9daf5b9482d581f, 0x167f53e0ba7431b8}, + {0x447600000027552e, 0xdcb8009a43480020, 0x6f7ee9ce4a6e8b59, 0xb10330b7c0a95bc6, 0x6140b1fcfb1e54b7, 0x0381be097f0bb4e1}, + {0x7588ffffffd8557d, 0x41f3ff646e0bffdf, 0xf7b1e8d2ac426aca, 0xb3741acd32dbb6f8, 0xe9daf5b9482d581f, 0x167f53e0ba7431b8}, }, { - fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, - fe{0, 0, 0, 0, 0, 0}, + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + {0, 0, 0, 0, 0, 0}, }, { - fe{0, 0, 0, 0, 0, 0}, - fe{0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, + {0, 0, 0, 0, 0, 0}, }, }, { { - fe{0x96d8f684bdfc77be, 0xb530e4f43b66d0e2, 0x184a88ff379652fd, 0x57cb23ecfae804e1, 0x0fd2e39eada3eba9, 0x08c8055e31c5d5c3}, - fe{0x96d8f684bdfc77be, 0xb530e4f43b66d0e2, 0x184a88ff379652fd, 0x57cb23ecfae804e1, 0x0fd2e39eada3eba9, 0x08c8055e31c5d5c3}, + {0x96d8f684bdfc77be, 0xb530e4f43b66d0e2, 0x184a88ff379652fd, 0x57cb23ecfae804e1, 0x0fd2e39eada3eba9, 0x08c8055e31c5d5c3}, + {0x96d8f684bdfc77be, 0xb530e4f43b66d0e2, 0x184a88ff379652fd, 0x57cb23ecfae804e1, 0x0fd2e39eada3eba9, 0x08c8055e31c5d5c3}, }, { - fe{0, 0, 0, 0, 0, 0}, - fe{0xbf0a71c71c91b406, 0x4d6d55d28b7638fd, 0x9d82f98e5f205aee, 0xa27aa27b1d1a18d5, 0x02c3b2b2d2938e86, 0x0c7d13420b09807f}, + {0, 0, 0, 0, 0, 0}, + {0xbf0a71c71c91b406, 0x4d6d55d28b7638fd, 0x9d82f98e5f205aee, 0xa27aa27b1d1a18d5, 0x02c3b2b2d2938e86, 0x0c7d13420b09807f}, }, { - fe{0xd7f9555555531c74, 0x21cffff748daaaa8, 0x5a9ad1866c9bbe46, 0x4870a2210221d251, 0x4a0db369c0a32af1, 0x02b1ccc429ff56af}, - fe{0xe205aaaaaaac8e37, 0xfcdc000768795556, 0x0c96011a8a1537dd, 0x1c06a963f163406e, 0x010df44c82a881e6, 0x174f45260f808feb}, + {0xd7f9555555531c74, 0x21cffff748daaaa8, 0x5a9ad1866c9bbe46, 0x4870a2210221d251, 0x4a0db369c0a32af1, 0x02b1ccc429ff56af}, + {0xe205aaaaaaac8e37, 0xfcdc000768795556, 0x0c96011a8a1537dd, 0x1c06a963f163406e, 0x010df44c82a881e6, 0x174f45260f808feb}, }, { - fe{0xa470bda12f67f35c, 0xc0fe38e23327b425, 0xc9d3d0f2c6f0678d, 0x1c55c9935b5a982e, 0x27f6c0e2f0746764, 0x117c5e6e28aa9054}, - fe{0, 0, 0, 0, 0, 0}, + {0xa470bda12f67f35c, 0xc0fe38e23327b425, 0xc9d3d0f2c6f0678d, 0x1c55c9935b5a982e, 0x27f6c0e2f0746764, 0x117c5e6e28aa9054}, + {0, 0, 0, 0, 0, 0}, }, }, { { - fe{0x0162fffffa765adf, 0x8f7bea480083fb75, 0x561b3c2259e93611, 0x11e19fc1a9c875d5, 0xca713efc00367660, 0x03c6a03d41da1151}, - fe{0x0162fffffa765adf, 0x8f7bea480083fb75, 0x561b3c2259e93611, 0x11e19fc1a9c875d5, 0xca713efc00367660, 0x03c6a03d41da1151}, + {0x0162fffffa765adf, 0x8f7bea480083fb75, 0x561b3c2259e93611, 0x11e19fc1a9c875d5, 0xca713efc00367660, 0x03c6a03d41da1151}, + {0x0162fffffa765adf, 0x8f7bea480083fb75, 0x561b3c2259e93611, 0x11e19fc1a9c875d5, 0xca713efc00367660, 0x03c6a03d41da1151}, }, { - fe{0, 0, 0, 0, 0, 0}, - fe{0x5db0fffffd3b02c5, 0xd713f52358ebfdba, 0x5ea60761a84d161a, 0xbb2c75a34ea6c44a, 0x0ac6735921c1119b, 0x0ee3d913bdacfbf6}, + {0, 0, 0, 0, 0, 0}, + {0x5db0fffffd3b02c5, 0xd713f52358ebfdba, 0x5ea60761a84d161a, 0xbb2c75a34ea6c44a, 0x0ac6735921c1119b, 0x0ee3d913bdacfbf6}, }, { - fe{0x66b10000003affc5, 0xcb1400e764ec0030, 0xa73e5eb56fa5d106, 0x8984c913a0fe09a9, 0x11e10afb78ad7f13, 0x05429d0e3e918f52}, - fe{0x534dffffffc4aae6, 0x5397ff174c67ffcf, 0xbff273eb870b251d, 0xdaf2827152870915, 0x393a9cbaca9e2dc3, 0x14be74dbfaee5748}, + {0x66b10000003affc5, 0xcb1400e764ec0030, 0xa73e5eb56fa5d106, 0x8984c913a0fe09a9, 0x11e10afb78ad7f13, 0x05429d0e3e918f52}, + {0x534dffffffc4aae6, 0x5397ff174c67ffcf, 0xbff273eb870b251d, 0xdaf2827152870915, 0x393a9cbaca9e2dc3, 0x14be74dbfaee5748}, }, { - fe{0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, - fe{0, 0, 0, 0, 0, 0}, + {0x760900000002fffd, 0xebf4000bc40c0002, 0x5f48985753c758ba, 0x77ce585370525745, 0x5c071a97a256ec6d, 0x15f65ec3fa80e493}, + {0, 0, 0, 0, 0, 0}, }, }, } diff --git a/crypto/bls12381/pairing.go b/crypto/bls12381/pairing.go index d292d7c3a5..abdc0a8790 100644 --- a/crypto/bls12381/pairing.go +++ b/crypto/bls12381/pairing.go @@ -1,19 +1,3 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 type pair struct { @@ -35,8 +19,8 @@ type Engine struct { pairs []pair } -// NewPairingEngine creates new pairing engine instance. -func NewPairingEngine() *Engine { +// NewEngine creates new pairing engine insteace. +func NewEngine() *Engine { fp2 := newFp2() fp6 := newFp6(fp2) fp12 := newFp12(fp6) @@ -52,24 +36,25 @@ func NewPairingEngine() *Engine { } type pairingEngineTemp struct { - t2 [10]*fe2 - t12 [9]fe12 + t2 [9]*fe2 + t12 [3]fe12 } func newEngineTemp() pairingEngineTemp { - t2 := [10]*fe2{} - for i := 0; i < 10; i++ { + t2 := [9]*fe2{} + for i := 0; i < len(t2); i++ { t2[i] = &fe2{} } - t12 := [9]fe12{} + t12 := [3]fe12{} return pairingEngineTemp{t2, t12} } // AddPair adds a g1, g2 point pair to pairing engine func (e *Engine) AddPair(g1 *PointG1, g2 *PointG2) *Engine { p := newPair(g1, g2) - if !e.isZero(p) { - e.affine(p) + if !(e.G1.IsZero(p.g1) || e.G2.IsZero(p.g2)) { + e.G1.Affine(p.g1) + e.G2.Affine(p.g2) e.pairs = append(e.pairs, p) } return e @@ -77,8 +62,9 @@ func (e *Engine) AddPair(g1 *PointG1, g2 *PointG2) *Engine { // AddPairInv adds a G1, G2 point pair to pairing engine. G1 point is negated. func (e *Engine) AddPairInv(g1 *PointG1, g2 *PointG2) *Engine { - e.G1.Neg(g1, g1) - e.AddPair(g1, g2) + ng1 := e.G1.New().Set(g1) + e.G1.Neg(ng1, g1) + e.AddPair(ng1, g2) return e } @@ -88,170 +74,246 @@ func (e *Engine) Reset() *Engine { return e } -func (e *Engine) isZero(p pair) bool { - return e.G1.IsZero(p.g1) || e.G2.IsZero(p.g2) -} +func (e *Engine) double(f *fe12, r *PointG2, k int) { + fp2, t := e.fp2, e.t2 -func (e *Engine) affine(p pair) { - e.G1.Affine(p.g1) - e.G2.Affine(p.g2) -} - -func (e *Engine) doublingStep(coeff *[3]fe2, r *PointG2) { - // Adaptation of Formula 3 in https://eprint.iacr.org/2010/526.pdf - fp2 := e.fp2 - t := e.t2 fp2.mul(t[0], &r[0], &r[1]) - fp2.mulByFq(t[0], t[0], twoInv) + fp2.mul0(t[0], t[0], twoInv) fp2.square(t[1], &r[1]) fp2.square(t[2], &r[2]) - fp2.double(t[7], t[2]) - fp2.add(t[7], t[7], t[2]) + fp2Double(t[7], t[2]) + fp2AddAssign(t[7], t[2]) fp2.mulByB(t[3], t[7]) - fp2.double(t[4], t[3]) - fp2.add(t[4], t[4], t[3]) - fp2.add(t[5], t[1], t[4]) - fp2.mulByFq(t[5], t[5], twoInv) - fp2.add(t[6], &r[1], &r[2]) - fp2.square(t[6], t[6]) - fp2.add(t[7], t[2], t[1]) - fp2.sub(t[6], t[6], t[7]) - fp2.sub(&coeff[0], t[3], t[1]) + fp2Double(t[4], t[3]) + fp2AddAssign(t[4], t[3]) + fp2Add(t[5], t[1], t[4]) + fp2.mul0(t[5], t[5], twoInv) + fp2Add(t[6], &r[1], &r[2]) + fp2.squareAssign(t[6]) + fp2Add(t[7], t[2], t[1]) + fp2SubAssign(t[6], t[7]) + + fp2Sub(t[8], t[3], t[1]) + fp2.square(t[7], &r[0]) - fp2.sub(t[4], t[1], t[4]) + fp2Sub(t[4], t[1], t[4]) fp2.mul(&r[0], t[4], t[0]) fp2.square(t[2], t[3]) - fp2.double(t[3], t[2]) - fp2.add(t[3], t[3], t[2]) - fp2.square(t[5], t[5]) - fp2.sub(&r[1], t[5], t[3]) + fp2Double(t[3], t[2]) + fp2AddAssign(t[3], t[2]) + fp2.squareAssign(t[5]) + fp2Sub(&r[1], t[5], t[3]) fp2.mul(&r[2], t[1], t[6]) - fp2.double(t[0], t[7]) - fp2.add(&coeff[1], t[0], t[7]) - fp2.neg(&coeff[2], t[6]) + fp2Double(t[0], t[7]) + + fp2AddAssign(t[0], t[7]) + fp2Neg(t[6], t[6]) + + // line eval + e.fp2.mul0Assign(t[6], &e.pairs[k].g1[1]) + e.fp2.mul0Assign(t[0], &e.pairs[k].g1[0]) + e.fp12.mul014(f, t[8], t[0], t[6]) + } -func (e *Engine) additionStep(coeff *[3]fe2, r, q *PointG2) { - // Algorithm 12 in https://eprint.iacr.org/2010/526.pdf - fp2 := e.fp2 - t := e.t2 - fp2.mul(t[0], &q[1], &r[2]) - fp2.neg(t[0], t[0]) - fp2.add(t[0], t[0], &r[1]) - fp2.mul(t[1], &q[0], &r[2]) - fp2.neg(t[1], t[1]) - fp2.add(t[1], t[1], &r[0]) +func (e *Engine) add(f *fe12, r *PointG2, k int) { + fp2, t := e.fp2, e.t2 + + fp2.mul(t[0], &e.pairs[k].g2[1], &r[2]) + fp2Neg(t[0], t[0]) + fp2AddAssign(t[0], &r[1]) + fp2.mul(t[1], &e.pairs[k].g2[0], &r[2]) + fp2Neg(t[1], t[1]) + fp2AddAssign(t[1], &r[0]) fp2.square(t[2], t[0]) fp2.square(t[3], t[1]) fp2.mul(t[4], t[1], t[3]) fp2.mul(t[2], &r[2], t[2]) - fp2.mul(t[3], &r[0], t[3]) - fp2.double(t[5], t[3]) - fp2.sub(t[5], t[4], t[5]) - fp2.add(t[5], t[5], t[2]) + fp2.mulAssign(t[3], &r[0]) + fp2Double(t[5], t[3]) + fp2Sub(t[5], t[4], t[5]) + fp2AddAssign(t[5], t[2]) fp2.mul(&r[0], t[1], t[5]) - fp2.sub(t[2], t[3], t[5]) - fp2.mul(t[2], t[2], t[0]) - fp2.mul(t[3], &r[1], t[4]) - fp2.sub(&r[1], t[2], t[3]) - fp2.mul(&r[2], &r[2], t[4]) - fp2.mul(t[2], t[1], &q[1]) - fp2.mul(t[3], t[0], &q[0]) - fp2.sub(&coeff[0], t[3], t[2]) - fp2.neg(&coeff[1], t[0]) - coeff[2].set(t[1]) + fp2SubAssign(t[3], t[5]) + fp2.mulAssign(t[3], t[0]) + fp2.mul(t[2], &r[1], t[4]) + fp2Sub(&r[1], t[3], t[2]) + fp2.mulAssign(&r[2], t[4]) + fp2.mul(t[2], t[1], &e.pairs[k].g2[1]) + fp2.mul(t[3], t[0], &e.pairs[k].g2[0]) + + fp2SubAssign(t[3], t[2]) + fp2Neg(t[0], t[0]) + + // line eval + e.fp2.mul0Assign(t[1], &e.pairs[k].g1[1]) + e.fp2.mul0Assign(t[0], &e.pairs[k].g1[0]) + e.fp12.mul014(f, t[3], t[0], t[1]) } -func (e *Engine) preCompute(ellCoeffs *[68][3]fe2, twistPoint *PointG2) { - // Algorithm 5 in https://eprint.iacr.org/2019/077.pdf - if e.G2.IsZero(twistPoint) { - return - } - r := new(PointG2).Set(twistPoint) - j := 0 - for i := x.BitLen() - 2; i >= 0; i-- { - e.doublingStep(&ellCoeffs[j], r) - if x.Bit(i) != 0 { - j++ - ellCoeffs[j] = fe6{} - e.additionStep(&ellCoeffs[j], r, twistPoint) +func (e *Engine) nDoubleAdd(f *fe12, r []PointG2, n int) { + for i := 0; i < n; i++ { + e.fp12.squareAssign(f) + for j := 0; j < len(e.pairs); j++ { + e.double(f, &r[j], j) + } + } + for j := 0; j < len(e.pairs); j++ { + e.add(f, &r[j], j) + } +} + +func (e *Engine) nDouble(f *fe12, r []PointG2, n int) { + for i := 0; i < n; i++ { + e.fp12.squareAssign(f) + for j := 0; j < len(e.pairs); j++ { + e.double(f, &r[j], j) } - j++ } } func (e *Engine) millerLoop(f *fe12) { - pairs := e.pairs - ellCoeffs := make([][68][3]fe2, len(pairs)) - for i := 0; i < len(pairs); i++ { - e.preCompute(&ellCoeffs[i], pairs[i].g2) - } - fp12, fp2 := e.fp12, e.fp2 - t := e.t2 f.one() - j := 0 - for i := 62; /* x.BitLen() - 2 */ i >= 0; i-- { - if i != 62 { - fp12.square(f, f) - } - for i := 0; i <= len(pairs)-1; i++ { - fp2.mulByFq(t[0], &ellCoeffs[i][j][2], &pairs[i].g1[1]) - fp2.mulByFq(t[1], &ellCoeffs[i][j][1], &pairs[i].g1[0]) - fp12.mulBy014Assign(f, &ellCoeffs[i][j][0], t[1], t[0]) - } - if x.Bit(i) != 0 { - j++ - for i := 0; i <= len(pairs)-1; i++ { - fp2.mulByFq(t[0], &ellCoeffs[i][j][2], &pairs[i].g1[1]) - fp2.mulByFq(t[1], &ellCoeffs[i][j][1], &pairs[i].g1[0]) - fp12.mulBy014Assign(f, &ellCoeffs[i][j][0], t[1], t[0]) - } - } - j++ + + r := make([]PointG2, len(e.pairs)) + for i := 0; i < len(e.pairs); i++ { + r[i].Set(e.pairs[i].g2) } - fp12.conjugate(f, f) + + for j := 0; j < len(e.pairs); j++ { + e.double(f, &r[j], j) + } + for j := 0; j < len(e.pairs); j++ { + e.add(f, &r[j], j) + } + + e.nDoubleAdd(f, r, 2) + e.nDoubleAdd(f, r, 3) + e.nDoubleAdd(f, r, 9) + e.nDoubleAdd(f, r, 32) + e.nDouble(f, r, 16) + + fp12Conjugate(f, f) } +// exp raises element by x = -15132376222941642752 func (e *Engine) exp(c, a *fe12) { - fp12 := e.fp12 - fp12.cyclotomicExp(c, a, x) - fp12.conjugate(c, c) + c.set(a) + e.fp12.cyclotomicSquare(c) // (a ^ 2) + + // (a ^ (2 + 1)) ^ (2 ^ 2) = a ^ 12 + e.fp12.mulAssign(c, a) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + + // (a ^ (12 + 1)) ^ (2 ^ 3) = a ^ 104 + e.fp12.mulAssign(c, a) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + + // (a ^ (104 + 1)) ^ (2 ^ 9) = a ^ 53760 + e.fp12.mulAssign(c, a) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + // (a ^ (53760 + 1)) ^ (2 ^ 32) = a ^ 230901736800256 + e.fp12.mulAssign(c, a) + for i := 0; i < 32; i++ { + e.fp12.cyclotomicSquare(c) + } + + // (a ^ (230901736800256 + 1)) ^ (2 ^ 16) = a ^ 15132376222941642752 + e.fp12.mulAssign(c, a) + for i := 0; i < 16; i++ { + e.fp12.cyclotomicSquare(c) + } + // invert chain result since x is negative + fp12Conjugate(c, c) +} + +// expDrop raises element by x = -15132376222941642752 / 2 +func (e *Engine) expDrop(c, a *fe12) { + c.set(a) + e.fp12.cyclotomicSquare(c) // (a ^ 2) + + // (a ^ (2 + 1)) ^ (2 ^ 2) = a ^ 12 + e.fp12.mulAssign(c, a) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + + // (a ^ (12 + 1)) ^ (2 ^ 3) = a ^ 104 + e.fp12.mulAssign(c, a) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + + // (a ^ (104 + 1)) ^ (2 ^ 9) = a ^ 53760 + e.fp12.mulAssign(c, a) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + e.fp12.cyclotomicSquare(c) + // (a ^ (53760 + 1)) ^ (2 ^ 32) = a ^ 230901736800256 + e.fp12.mulAssign(c, a) + for i := 0; i < 32; i++ { + e.fp12.cyclotomicSquare(c) + } + + // (a ^ (230901736800256 + 1)) ^ (2 ^ 16) = a ^ 15132376222941642752 + e.fp12.mulAssign(c, a) + for i := 0; i < 15; i++ { + e.fp12.cyclotomicSquare(c) + } + // invert chain result since x is negative + fp12Conjugate(c, c) } func (e *Engine) finalExp(f *fe12) { - fp12 := e.fp12 t := e.t12 + // Efficient Final Exponentiation via Cyclotomic Structure for Pairings over Families of Elliptic Curves + // https: //eprint.iacr.org/2020/875.pdf + // easy part - fp12.frobeniusMap(&t[0], f, 6) - fp12.inverse(&t[1], f) - fp12.mul(&t[2], &t[0], &t[1]) - t[1].set(&t[2]) - fp12.frobeniusMapAssign(&t[2], 2) - fp12.mulAssign(&t[2], &t[1]) - fp12.cyclotomicSquare(&t[1], &t[2]) - fp12.conjugate(&t[1], &t[1]) + fp12Conjugate(&t[0], f) + e.fp12.inverse(f, f) + e.fp12.mulAssign(&t[0], f) + f.set(&t[0]) + e.fp12.frobeniusMap2(f) + e.fp12.mulAssign(f, &t[0]) + // hard part - e.exp(&t[3], &t[2]) - fp12.cyclotomicSquare(&t[4], &t[3]) - fp12.mul(&t[5], &t[1], &t[3]) - e.exp(&t[1], &t[5]) + t[0].set(f) + e.fp12.cyclotomicSquare(&t[0]) + e.expDrop(&t[1], &t[0]) + fp12Conjugate(&t[2], f) + e.fp12.mulAssign(&t[1], &t[2]) + e.exp(&t[2], &t[1]) + fp12Conjugate(&t[1], &t[1]) + e.fp12.mulAssign(&t[1], &t[2]) + e.exp(&t[2], &t[1]) + e.fp12.frobeniusMap1(&t[1]) + e.fp12.mulAssign(&t[1], &t[2]) + e.fp12.mulAssign(f, &t[0]) e.exp(&t[0], &t[1]) - e.exp(&t[6], &t[0]) - fp12.mulAssign(&t[6], &t[4]) - e.exp(&t[4], &t[6]) - fp12.conjugate(&t[5], &t[5]) - fp12.mulAssign(&t[4], &t[5]) - fp12.mulAssign(&t[4], &t[2]) - fp12.conjugate(&t[5], &t[2]) - fp12.mulAssign(&t[1], &t[2]) - fp12.frobeniusMapAssign(&t[1], 3) - fp12.mulAssign(&t[6], &t[5]) - fp12.frobeniusMapAssign(&t[6], 1) - fp12.mulAssign(&t[3], &t[0]) - fp12.frobeniusMapAssign(&t[3], 2) - fp12.mulAssign(&t[3], &t[1]) - fp12.mulAssign(&t[3], &t[6]) - fp12.mul(f, &t[3], &t[4]) + e.exp(&t[2], &t[0]) + t[0].set(&t[1]) + e.fp12.frobeniusMap2(&t[0]) + fp12Conjugate(&t[1], &t[1]) + e.fp12.mulAssign(&t[1], &t[2]) + e.fp12.mulAssign(&t[1], &t[0]) + e.fp12.mulAssign(f, &t[1]) } func (e *Engine) calculate() *fe12 { diff --git a/crypto/bls12381/pairing_test.go b/crypto/bls12381/pairing_test.go index 77676fe9b1..a46545f1d7 100644 --- a/crypto/bls12381/pairing_test.go +++ b/crypto/bls12381/pairing_test.go @@ -3,28 +3,27 @@ package bls12381 import ( "math/big" "testing" - - "github.com/ethereum/go-ethereum/common" ) func TestPairingExpected(t *testing.T) { - bls := NewPairingEngine() + bls := NewEngine() G1, G2 := bls.G1, bls.G2 GT := bls.GT() expected, err := GT.FromBytes( - common.FromHex("" + - "0f41e58663bf08cf068672cbd01a7ec73baca4d72ca93544deff686bfd6df543d48eaa24afe47e1efde449383b676631" + - "04c581234d086a9902249b64728ffd21a189e87935a954051c7cdba7b3872629a4fafc05066245cb9108f0242d0fe3ef" + - "03350f55a7aefcd3c31b4fcb6ce5771cc6a0e9786ab5973320c806ad360829107ba810c5a09ffdd9be2291a0c25a99a2" + - "11b8b424cd48bf38fcef68083b0b0ec5c81a93b330ee1a677d0d15ff7b984e8978ef48881e32fac91b93b47333e2ba57" + - "06fba23eb7c5af0d9f80940ca771b6ffd5857baaf222eb95a7d2809d61bfe02e1bfd1b68ff02f0b8102ae1c2d5d5ab1a" + - "19f26337d205fb469cd6bd15c3d5a04dc88784fbb3d0b2dbdea54d43b2b73f2cbb12d58386a8703e0f948226e47ee89d" + - "018107154f25a764bd3c79937a45b84546da634b8f6be14a8061e55cceba478b23f7dacaa35c8ca78beae9624045b4b6" + - "01b2f522473d171391125ba84dc4007cfbf2f8da752f7c74185203fcca589ac719c34dffbbaad8431dad1c1fb597aaa5" + - "193502b86edb8857c273fa075a50512937e0794e1e65a7617c90d8bd66065b1fffe51d7a579973b1315021ec3c19934f" + - "1368bb445c7c2d209703f239689ce34c0378a68e72a6b3b216da0e22a5031b54ddff57309396b38c881c4c849ec23e87" + - "089a1c5b46e5110b86750ec6a532348868a84045483c92b7af5af689452eafabf1a8943e50439f1d59882a98eaa0170f" + - "1250ebd871fc0a92a7b2d83168d0d727272d441befa15c503dd8e90ce98db3e7b6d194f60839c508a84305aaca1789b6", + fromHex( + fpByteSize, + "0x0f41e58663bf08cf068672cbd01a7ec73baca4d72ca93544deff686bfd6df543d48eaa24afe47e1efde449383b676631", + "0x04c581234d086a9902249b64728ffd21a189e87935a954051c7cdba7b3872629a4fafc05066245cb9108f0242d0fe3ef", + "0x03350f55a7aefcd3c31b4fcb6ce5771cc6a0e9786ab5973320c806ad360829107ba810c5a09ffdd9be2291a0c25a99a2", + "0x11b8b424cd48bf38fcef68083b0b0ec5c81a93b330ee1a677d0d15ff7b984e8978ef48881e32fac91b93b47333e2ba57", + "0x06fba23eb7c5af0d9f80940ca771b6ffd5857baaf222eb95a7d2809d61bfe02e1bfd1b68ff02f0b8102ae1c2d5d5ab1a", + "0x19f26337d205fb469cd6bd15c3d5a04dc88784fbb3d0b2dbdea54d43b2b73f2cbb12d58386a8703e0f948226e47ee89d", + "0x018107154f25a764bd3c79937a45b84546da634b8f6be14a8061e55cceba478b23f7dacaa35c8ca78beae9624045b4b6", + "0x01b2f522473d171391125ba84dc4007cfbf2f8da752f7c74185203fcca589ac719c34dffbbaad8431dad1c1fb597aaa5", + "0x193502b86edb8857c273fa075a50512937e0794e1e65a7617c90d8bd66065b1fffe51d7a579973b1315021ec3c19934f", + "0x1368bb445c7c2d209703f239689ce34c0378a68e72a6b3b216da0e22a5031b54ddff57309396b38c881c4c849ec23e87", + "0x089a1c5b46e5110b86750ec6a532348868a84045483c92b7af5af689452eafabf1a8943e50439f1d59882a98eaa0170f", + "0x1250ebd871fc0a92a7b2d83168d0d727272d441befa15c503dd8e90ce98db3e7b6d194f60839c508a84305aaca1789b6", ), ) if err != nil { @@ -32,7 +31,7 @@ func TestPairingExpected(t *testing.T) { } r := bls.AddPair(G1.One(), G2.One()).Result() if !r.Equal(expected) { - t.Fatal("bad pairing") + t.Fatal("expected pairing failed") } if !GT.IsValid(r) { t.Fatal("element is not in correct subgroup") @@ -40,7 +39,7 @@ func TestPairingExpected(t *testing.T) { } func TestPairingNonDegeneracy(t *testing.T) { - bls := NewPairingEngine() + bls := NewEngine() G1, G2 := bls.G1, bls.G2 g1Zero, g2Zero, g1One, g2One := G1.Zero(), G2.Zero(), G1.One(), G2.One() GT := bls.GT() @@ -89,19 +88,20 @@ func TestPairingNonDegeneracy(t *testing.T) { bls.Reset() { expected, err := GT.FromBytes( - common.FromHex("" + - "0f41e58663bf08cf068672cbd01a7ec73baca4d72ca93544deff686bfd6df543d48eaa24afe47e1efde449383b676631" + - "04c581234d086a9902249b64728ffd21a189e87935a954051c7cdba7b3872629a4fafc05066245cb9108f0242d0fe3ef" + - "03350f55a7aefcd3c31b4fcb6ce5771cc6a0e9786ab5973320c806ad360829107ba810c5a09ffdd9be2291a0c25a99a2" + - "11b8b424cd48bf38fcef68083b0b0ec5c81a93b330ee1a677d0d15ff7b984e8978ef48881e32fac91b93b47333e2ba57" + - "06fba23eb7c5af0d9f80940ca771b6ffd5857baaf222eb95a7d2809d61bfe02e1bfd1b68ff02f0b8102ae1c2d5d5ab1a" + - "19f26337d205fb469cd6bd15c3d5a04dc88784fbb3d0b2dbdea54d43b2b73f2cbb12d58386a8703e0f948226e47ee89d" + - "018107154f25a764bd3c79937a45b84546da634b8f6be14a8061e55cceba478b23f7dacaa35c8ca78beae9624045b4b6" + - "01b2f522473d171391125ba84dc4007cfbf2f8da752f7c74185203fcca589ac719c34dffbbaad8431dad1c1fb597aaa5" + - "193502b86edb8857c273fa075a50512937e0794e1e65a7617c90d8bd66065b1fffe51d7a579973b1315021ec3c19934f" + - "1368bb445c7c2d209703f239689ce34c0378a68e72a6b3b216da0e22a5031b54ddff57309396b38c881c4c849ec23e87" + - "089a1c5b46e5110b86750ec6a532348868a84045483c92b7af5af689452eafabf1a8943e50439f1d59882a98eaa0170f" + - "1250ebd871fc0a92a7b2d83168d0d727272d441befa15c503dd8e90ce98db3e7b6d194f60839c508a84305aaca1789b6", + fromHex( + fpByteSize, + "0x0f41e58663bf08cf068672cbd01a7ec73baca4d72ca93544deff686bfd6df543d48eaa24afe47e1efde449383b676631", + "0x04c581234d086a9902249b64728ffd21a189e87935a954051c7cdba7b3872629a4fafc05066245cb9108f0242d0fe3ef", + "0x03350f55a7aefcd3c31b4fcb6ce5771cc6a0e9786ab5973320c806ad360829107ba810c5a09ffdd9be2291a0c25a99a2", + "0x11b8b424cd48bf38fcef68083b0b0ec5c81a93b330ee1a677d0d15ff7b984e8978ef48881e32fac91b93b47333e2ba57", + "0x06fba23eb7c5af0d9f80940ca771b6ffd5857baaf222eb95a7d2809d61bfe02e1bfd1b68ff02f0b8102ae1c2d5d5ab1a", + "0x19f26337d205fb469cd6bd15c3d5a04dc88784fbb3d0b2dbdea54d43b2b73f2cbb12d58386a8703e0f948226e47ee89d", + "0x018107154f25a764bd3c79937a45b84546da634b8f6be14a8061e55cceba478b23f7dacaa35c8ca78beae9624045b4b6", + "0x01b2f522473d171391125ba84dc4007cfbf2f8da752f7c74185203fcca589ac719c34dffbbaad8431dad1c1fb597aaa5", + "0x193502b86edb8857c273fa075a50512937e0794e1e65a7617c90d8bd66065b1fffe51d7a579973b1315021ec3c19934f", + "0x1368bb445c7c2d209703f239689ce34c0378a68e72a6b3b216da0e22a5031b54ddff57309396b38c881c4c849ec23e87", + "0x089a1c5b46e5110b86750ec6a532348868a84045483c92b7af5af689452eafabf1a8943e50439f1d59882a98eaa0170f", + "0x1250ebd871fc0a92a7b2d83168d0d727272d441befa15c503dd8e90ce98db3e7b6d194f60839c508a84305aaca1789b6", ), ) if err != nil { @@ -113,13 +113,13 @@ func TestPairingNonDegeneracy(t *testing.T) { bls.AddPair(g1One, g2One) e := bls.Result() if !e.Equal(expected) { - t.Fatal("bad pairing") + t.Fatal("pairing failed") } } } func TestPairingBilinearity(t *testing.T) { - bls := NewPairingEngine() + bls := NewEngine() g1, g2 := bls.G1, bls.G2 gt := bls.GT() // e(a*G1, b*G2) = e(G1, G2)^c @@ -129,50 +129,50 @@ func TestPairingBilinearity(t *testing.T) { G1, G2 := g1.One(), g2.One() e0 := bls.AddPair(G1, G2).Result() P1, P2 := g1.New(), g2.New() - g1.MulScalar(P1, G1, a) - g2.MulScalar(P2, G2, b) + g1.MulScalarBig(P1, G1, a) + g2.MulScalarBig(P2, G2, b) e1 := bls.AddPair(P1, P2).Result() gt.Exp(e0, e0, c) if !e0.Equal(e1) { - t.Fatal("bad pairing, 1") + t.Fatal("pairing failed") } } - // e(a * G1, b * G2) = e((a + b) * G1, G2) + // e(a * G1, b * G2) = e((a * b) * G1, G2) { // scalars a, b := big.NewInt(17), big.NewInt(117) c := new(big.Int).Mul(a, b) // LHS G1, G2 := g1.One(), g2.One() - g1.MulScalar(G1, G1, c) + g1.MulScalarBig(G1, G1, c) bls.AddPair(G1, G2) // RHS P1, P2 := g1.One(), g2.One() - g1.MulScalar(P1, P1, a) - g2.MulScalar(P2, P2, b) + g1.MulScalarBig(P1, P1, a) + g2.MulScalarBig(P2, P2, b) bls.AddPairInv(P1, P2) // should be one if !bls.Check() { - t.Fatal("bad pairing, 2") + t.Fatal("pairing failed") } } - // e(a * G1, b * G2) = e((a + b) * G1, G2) + // e(a * G1, b * G2) = e(G1, (a * b) * G2) { // scalars a, b := big.NewInt(17), big.NewInt(117) c := new(big.Int).Mul(a, b) // LHS G1, G2 := g1.One(), g2.One() - g2.MulScalar(G2, G2, c) + g2.MulScalarBig(G2, G2, c) bls.AddPair(G1, G2) // RHS H1, H2 := g1.One(), g2.One() - g1.MulScalar(H1, H1, a) - g2.MulScalar(H2, H2, b) + g1.MulScalarBig(H1, H1, a) + g2.MulScalarBig(H2, H2, b) bls.AddPairInv(H1, H2) // should be one if !bls.Check() { - t.Fatal("bad pairing, 3") + t.Fatal("pairing failed") } } } @@ -180,17 +180,17 @@ func TestPairingBilinearity(t *testing.T) { func TestPairingMulti(t *testing.T) { // e(G1, G2) ^ t == e(a01 * G1, a02 * G2) * e(a11 * G1, a12 * G2) * ... * e(an1 * G1, an2 * G2) // where t = sum(ai1 * ai2) - bls := NewPairingEngine() + bls := NewEngine() g1, g2 := bls.G1, bls.G2 numOfPair := 100 targetExp := new(big.Int) // RHS for i := 0; i < numOfPair; i++ { // (ai1 * G1, ai2 * G2) - a1, a2 := randScalar(q), randScalar(q) + a1, a2 := randScalar(qBig), randScalar(qBig) P1, P2 := g1.One(), g2.One() - g1.MulScalar(P1, P1, a1) - g2.MulScalar(P2, P2, a2) + g1.MulScalarBig(P1, P1, a1) + g2.MulScalarBig(P2, P2, a2) bls.AddPair(P1, P2) // accumulate targetExp // t += (ai1 * ai2) @@ -200,7 +200,7 @@ func TestPairingMulti(t *testing.T) { // LHS // e(t * G1, G2) T1, T2 := g1.One(), g2.One() - g1.MulScalar(T1, T1, targetExp) + g1.MulScalarBig(T1, T1, targetExp) bls.AddPairInv(T1, T2) if !bls.Check() { t.Fatal("fail multi pairing") @@ -208,7 +208,7 @@ func TestPairingMulti(t *testing.T) { } func TestPairingEmpty(t *testing.T) { - bls := NewPairingEngine() + bls := NewEngine() if !bls.Check() { t.Fatal("empty check should be accepted") } @@ -218,7 +218,7 @@ func TestPairingEmpty(t *testing.T) { } func BenchmarkPairing(t *testing.B) { - bls := NewPairingEngine() + bls := NewEngine() g1, g2, gt := bls.G1, bls.G2, bls.GT() bls.AddPair(g1.One(), g2.One()) e := gt.New() @@ -228,3 +228,25 @@ func BenchmarkPairing(t *testing.B) { } _ = e } + +func BenchmarkMillerLoop(t *testing.B) { + bls := NewEngine() + g1, g2, gt := bls.G1, bls.G2, bls.GT() + bls.AddPair(g1.One(), g2.One()) + f := gt.New().one() + t.ResetTimer() + for i := 0; i < t.N; i++ { + bls.millerLoop(f) + } +} + +func BenchmarkFinalExp(t *testing.B) { + bls := NewEngine() + g1, g2, gt := bls.G1, bls.G2, bls.GT() + bls.AddPair(g1.One(), g2.One()) + f := gt.New().one() + t.ResetTimer() + for i := 0; i < t.N; i++ { + bls.finalExp(f) + } +} diff --git a/crypto/bls12381/swu.go b/crypto/bls12381/swu.go index e78753b240..0569ff0bb8 100644 --- a/crypto/bls12381/swu.go +++ b/crypto/bls12381/swu.go @@ -1,23 +1,7 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 // swuMapG1 is implementation of Simplified Shallue-van de Woestijne-Ulas Method -// follows the implementation at draft-irtf-cfrg-hash-to-curve-06. +// follows the implmentation at draft-irtf-cfrg-hash-to-curve-06. func swuMapG1(u *fe) (*fe, *fe) { var params = swuParamsForG1 var tv [4]*fe @@ -79,19 +63,19 @@ func swuMapG2(e *fp2, u *fe2) (*fe2, *fe2) { e.mul(tv[0], tv[0], params.z) e.square(tv[1], tv[0]) x1 := e.new() - e.add(x1, tv[0], tv[1]) + fp2Add(x1, tv[0], tv[1]) e.inverse(x1, x1) e1 := x1.isZero() - e.add(x1, x1, e.one()) + fp2Add(x1, x1, e.one()) if e1 { x1.set(params.zInv) } e.mul(x1, x1, params.minusBOverA) gx1 := e.new() e.square(gx1, x1) - e.add(gx1, gx1, params.a) + fp2Add(gx1, gx1, params.a) e.mul(gx1, gx1, x1) - e.add(gx1, gx1, params.b) + fp2Add(gx1, gx1, params.b) x2 := e.new() e.mul(x2, tv[0], x1) e.mul(tv[1], tv[0], tv[1]) @@ -107,9 +91,9 @@ func swuMapG2(e *fp2, u *fe2) (*fe2, *fe2) { y2.set(gx2) } y := e.new() - e.sqrt(y, y2) + e.sqrtBLST(y, y2) if y.sign() != u.sign() { - e.neg(y, y) + fp2Neg(y, y) } return x, y } diff --git a/crypto/bls12381/utils.go b/crypto/bls12381/utils.go index de8bf495fe..a5fb988e01 100644 --- a/crypto/bls12381/utils.go +++ b/crypto/bls12381/utils.go @@ -1,45 +1,13 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - package bls12381 import ( - "errors" "math/big" - - "github.com/ethereum/go-ethereum/common" ) func bigFromHex(hex string) *big.Int { - return new(big.Int).SetBytes(common.FromHex(hex)) -} - -// decodeFieldElement expects 64 byte input with zero top 16 bytes, -// returns lower 48 bytes. -func decodeFieldElement(in []byte) ([]byte, error) { - if len(in) != 64 { - return nil, errors.New("invalid field element length") + if len(hex) > 1 && hex[:2] == "0x" { + hex = hex[2:] } - // check top bytes - for i := 0; i < 16; i++ { - if in[i] != byte(0x00) { - return nil, errors.New("invalid field element top bytes") - } - } - out := make([]byte, 48) - copy(out[:], in[16:]) - return out, nil + n, _ := new(big.Int).SetString(hex, 16) + return n } diff --git a/crypto/bls12381/wnaf.go b/crypto/bls12381/wnaf.go new file mode 100644 index 0000000000..9ab2d57306 --- /dev/null +++ b/crypto/bls12381/wnaf.go @@ -0,0 +1,110 @@ +package bls12381 + +import ( + "math/big" +) + +type nafNumber []int + +func (n nafNumber) neg() { + for i := 0; i < len(n); i++ { + n[i] = -n[i] + } +} + +var bigZero = big.NewInt(0) +var bigOne = big.NewInt(1) + +func (e *Fr) toWNAF(w uint) nafNumber { + naf := nafNumber{} + if w == 0 { + return naf + } + windowSize, halfSize, mask := 1<<(w+1), 1<= halfSize { + nafSign = nafSign - windowSize + } + naf = append(naf, int(nafSign)) + if nafSign < 0 { + laddAssignFR(ee, z.setUint64(uint64(-nafSign))) + } else { + lsubAssignFR(ee, z.setUint64(uint64(nafSign))) + } + } else { + naf = append(naf, 0) + } + ee.div2() + } + + return naf +} + +func (e *Fr) fromWNAF(naf nafNumber, w uint) *Fr { + if w == 0 { + return e + } + l := (1 << (w - 1)) + table := make([]*Fr, l) + table[0] = new(Fr).One() + two := new(Fr).setUint64(2) + for i := 1; i < l; i++ { + table[i] = new(Fr) + table[i].Add(table[i-1], two) + } + acc := new(Fr).Zero() + for i := len(naf) - 1; i >= 0; i-- { + if naf[i] < 0 { + acc.Sub(acc, table[-naf[i]>>1]) + } else if naf[i] > 0 { + acc.Add(acc, table[naf[i]>>1]) + } + if i != 0 { + acc.Double(acc) + } + } + return e.Set(acc) +} + +// caution: does not cover negative case +func bigToWNAF(e *big.Int, w uint) nafNumber { + naf := nafNumber{} + if w == 0 { + return naf + } + windowSize := new(big.Int).Lsh(bigOne, uint(w+1)) + halfSize := new(big.Int).Rsh(windowSize, 1) + ee := new(big.Int).Abs(e) + for ee.Cmp(bigZero) != 0 { + if ee.Bit(0) == 1 { + nafSign := new(big.Int) + nafSign.Mod(ee, windowSize) + if nafSign.Cmp(halfSize) >= 0 { + nafSign.Sub(nafSign, windowSize) + } + naf = append(naf, int(nafSign.Int64())) + ee.Sub(ee, nafSign) + } else { + naf = append(naf, 0) + } + ee.Rsh(ee, 1) + } + return naf +} + +func bigFromWNAF(naf nafNumber) *big.Int { + acc := new(big.Int) + k := new(big.Int).Set(bigOne) + for i := 0; i < len(naf); i++ { + if naf[i] != 0 { + z := new(big.Int).Mul(k, big.NewInt(int64(naf[i]))) + acc.Add(acc, z) + } + k.Lsh(k, 1) + } + return acc +} diff --git a/crypto/bls12381/wnaf_test.go b/crypto/bls12381/wnaf_test.go new file mode 100644 index 0000000000..c8b8955cca --- /dev/null +++ b/crypto/bls12381/wnaf_test.go @@ -0,0 +1,67 @@ +package bls12381 + +import ( + "crypto/rand" + "math/big" + "testing" +) + +var maxWindowSize uint = 9 + +func TestWNAFBig(t *testing.T) { + var w uint + for w = 1; w <= maxWindowSize; w++ { + for i := 0; i < fuz; i++ { + e0, err := rand.Int(rand.Reader, new(big.Int).SetUint64(100)) + if err != nil { + t.Fatal(err) + } + n0 := bigToWNAF(e0, w) + e1 := bigFromWNAF(n0) + if e0.Cmp(e1) != 0 { + t.Fatal("wnaf conversion failed") + } + } + } +} + +func TestFrWNAF(t *testing.T) { + var w uint + for w = 1; w <= maxWindowSize; w++ { + for i := 0; i < fuz; i++ { + a0, _ := new(Fr).Rand(rand.Reader) + naf := a0.toWNAF(w) + a1 := new(Fr).fromWNAF(naf, w) + if !a0.Equal(a1) { + t.Fatal("wnaf conversion failed") + } + naf.neg() + a1.fromWNAF(naf, w) + a0.Neg(a0) + if !a0.Equal(a1) { + t.Fatal("negated wnaf conversion failed") + } + } + } +} + +func TestFrWNAFCrossAgainstBig(t *testing.T) { + var maxWindowSize uint = 20 + var w uint + for w = 1; w <= maxWindowSize; w++ { + for i := 0; i < fuz; i++ { + a, _ := new(Fr).Rand(rand.Reader) + aBig := a.ToBig() + naf1 := a.toWNAF(w) + naf2 := bigToWNAF(aBig, w) + if len(naf1) != len(naf2) { + t.Fatal("naf conversion failed", len(naf1), len(naf2)) + } + for i := 0; i < len(naf1); i++ { + if naf1[i] != naf2[i] { + t.Fatal("naf conversion failed", i) + } + } + } + } +}