crypto/kzg4844: add RecoverCells with systematic fast path (#35529)

Add RecoverCells, which returns all CellsPerBlob cells per blob from a
sufficient subset.

RecoverBlobs only exposes recovered blobs; serving or persisting the
extension cells of a sparse-blobpool transaction needs the full 128-cell
set. Add RecoverCells, which returns all CellsPerBlob cells per blob
from a sufficient subset.

When the full data domain (cell indices 0..DataPerBlob-1) is present
(the common case for pooled transactions) the blobs are a free
concatenation of the data cells and every cell follows from a systematic
extension via ComputeCells (~2-6ms/blob), skipping the KZG erasure
solve. Otherwise it falls back to the erasure recovery path
(~15-20ms/blob), which now surfaces the library's full extended cell set
instead of discarding it down to blobs as RecoverBlobs does. Both paths
return byte-identical cells in canonical order.

Cells only; cell proofs are never recomputed (callers retain the proofs
shipped with the transaction). Tested on both the gokzg and ckzg
backends, fast and slow paths, against the original cells.

Note: the same fast-path optimization could also be pushed down to gokzg
and ckzg. There are both arguments for (the primitive becomes faster in
some cases, same for the rest of cases) and against (it is not the role
of the crypto lib to be intelligent here), so I went with the wrapper
for now.

crypto/kzg4844/kzg4844.go

@@ -19,6 +19,7 @@ package kzg4844
 
 import (
 	"embed"
+	"encoding/binary"
 	"errors"
 	"hash"
 	"reflect"
@@ -259,23 +260,80 @@ func ComputeCells(blobs []Blob) ([]Cell, error) {
 // RecoverBlobs recovers blobs from the given cells and cell indices.
 // In order to successfully recover, at least DataPerBlob (64) cells must be provided.
 //
+// When the data cells (indices 0..DataPerBlob-1) are all present, the blobs are
+// by definition their concatenation, which is returned without any KZG work.
+// That is byte-identical to the erasure recovery for any input whose redundant
+// cells are consistent with the data, which is all a valid sidecar can produce.
+// Where they conflict, the data cells decide, whereas the erasure recovery
+// mixes the conflicting cells in and returns neither faithfully.
+//
 // For the layout of cells and cellIndices, please see [VerifyCells].
 func RecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
 	if err := validateCellIndices(cells, cellIndices); err != nil {
 		return nil, err
 	}
+	if blobs, ok := blobsFromDataCells(cells, cellIndices); ok {
+		return blobs, nil
+	}
 	if useCKZG.Load() {
 		return ckzgRecoverBlobs(cells, cellIndices)
 	}
 	return gokzgRecoverBlobs(cells, cellIndices)
 }
 
+// Field modulus of the BLS12-381 scalar field as big-endian 64-bit limbs,
+// most significant first:
+//
+//	0x73eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001
+//
+// TestFieldModulusLimbs pins these against the canonical definition.
+const (
+	bytesPerFieldElement = 32
+
+	frModulusW0 uint64 = 0x73eda753299d7d48
+	frModulusW1 uint64 = 0x3339d80809a1d805
+	frModulusW2 uint64 = 0x53bda402fffe5bfe
+	frModulusW3 uint64 = 0xffffffff00000001
+)
+
+// isCanonicalFieldElement reports whether the big-endian 32-byte scalar is a
+// canonical field element, i.e. strictly below the field modulus. It compares
+// limb by limb, so well-formed data resolves on the first comparison.
+func isCanonicalFieldElement(b []byte) bool {
+	if w := binary.BigEndian.Uint64(b[0:8]); w != frModulusW0 {
+		return w < frModulusW0
+	}
+	if w := binary.BigEndian.Uint64(b[8:16]); w != frModulusW1 {
+		return w < frModulusW1
+	}
+	if w := binary.BigEndian.Uint64(b[16:24]); w != frModulusW2 {
+		return w < frModulusW2
+	}
+	return binary.BigEndian.Uint64(b[24:32]) < frModulusW3
+}
+
+// isCanonicalCell reports whether every field element of the cell is canonical.
+func isCanonicalCell(cell *Cell) bool {
+	for i := 0; i < len(cell); i += bytesPerFieldElement {
+		if !isCanonicalFieldElement(cell[i : i+bytesPerFieldElement]) {
+			return false
+		}
+	}
+	return true
+}
+
 // blobsFromDataCells reconstructs blobs by concatenating their data cells (cell
 // indices 0..DataPerBlob-1, by definition the blob contents), with no KZG
-// involvement and no validation of the cell contents whatsoever (see
-// RecoverBlobsUnchecked for the contract). It accepts a strict subset of the
-// inputs RecoverBlobs accepts and returns identical bytes; on ok=false the
-// caller should fall back to RecoverBlobs.
+// involvement. It accepts a strict subset of the inputs the KZG recovery
+// accepts, returning the same bytes whenever the redundant cells are consistent
+// with the data; on ok=false the caller must fall back to that recovery, which
+// is also the authority on rejecting invalid input.
+//
+// Since the KZG library is bypassed, the checks it would perform while
+// deserializing happen here instead: cell indices must be well-formed, and
+// every input cell must hold canonical field elements. Cell contents are not
+// examined further -- nothing is verified against commitments or cell proofs,
+// and the redundancy is not checked for consistency with the data.
 //
 // For the layout of cells and cellIndices, see RecoverBlobs.
 func blobsFromDataCells(cells []Cell, cellIndices []uint64) ([]Blob, bool) {
@@ -299,6 +357,14 @@ func blobsFromDataCells(cells []Cell, cellIndices []uint64) ([]Blob, bool) {
 			return nil, false
 		}
 	}
+	// Likewise for the cell contents: every input cell must be canonical, the
+	// ignored tail cells included, so that declining and recovering through the
+	// KZG library cannot turn a rejection into a success.
+	for i := range cells {
+		if !isCanonicalCell(&cells[i]) {
+			return nil, false
+		}
+	}
 	blobCount := len(cells) / len(cellIndices)
 	blobs := make([]Blob, blobCount)
 	for b := range blobCount {
@@ -310,22 +376,40 @@ func blobsFromDataCells(cells []Cell, cellIndices []uint64) ([]Blob, bool) {
 	return blobs, true
 }
 
-// RecoverBlobsUnchecked is RecoverBlobs for callers that have already
-// established the cells' authenticity (e.g. via VerifyCells at ingest): when the
-// data cells are all present, the blobs are returned as their concatenation,
-// skipping the KZG erasure decode. The result is byte-identical to RecoverBlobs.
+// RecoverCells returns all CellsPerBlob cells for every blob represented by the
+// input cells, given a sufficient subset (at least DataPerBlob cells per blob).
+// When the full data domain (indices 0..DataPerBlob-1) is present, all cells
+// follow from a cheap systematic extension of the concatenated blobs
+// (ComputeCells), skipping the KZG erasure solve; otherwise it falls back to
+// full erasure recovery. For input whose redundant cells are consistent with the
+// data the two paths return byte-identical cells, in canonical index order per
+// blob. Cell proofs are never recomputed: callers that need proofs should retain
+// those shipped with the transaction.
 //
-// "Unchecked" refers to the cell contents: the concatenation validates nothing,
-// not even field-element canonicalness -- the one check RecoverBlobs performs
-// (neither verifies cell proofs or the commitment). Pass only cells whose
-// authenticity is already assured.
+// Both paths reject non-canonical field elements: the erasure path while
+// deserializing the input cells, the systematic path by checking them
+// explicitly. Neither path checks that redundant input cells are consistent
+// with the data: the underlying erasure decode assumes consistency and
+// silently returns wrong cells otherwise. Nothing is verified against
+// commitments or cell proofs -- the authenticity of every input cell must be
+// established by the caller (e.g. VerifyCells at ingest), which also
+// guarantees consistency.
 //
 // For the layout of cells and cellIndices, see RecoverBlobs.
-func RecoverBlobsUnchecked(cells []Cell, cellIndices []uint64) ([]Blob, error) {
-	if blobs, ok := blobsFromDataCells(cells, cellIndices); ok {
-		return blobs, nil
+func RecoverCells(cells []Cell, cellIndices []uint64) ([]Cell, error) {
+	if err := validateCellIndices(cells, cellIndices); err != nil {
+		return nil, err
 	}
-	return RecoverBlobs(cells, cellIndices)
+	// Fast path: the data cells are all present, so the blobs are a free
+	// concatenation and all cells follow from a systematic extension.
+	if blobs, ok := blobsFromDataCells(cells, cellIndices); ok {
+		return ComputeCells(blobs)
+	}
+	// Slow path: genuine erasure recovery from a non-data subset.
+	if useCKZG.Load() {
+		return ckzgRecoverCells(cells, cellIndices)
+	}
+	return gokzgRecoverCells(cells, cellIndices)
 }
 
 func validateCellIndices(cells []Cell, cellIndices []uint64) error {

crypto/kzg4844/kzg4844_ckzg_cgo.go

@@ -279,3 +279,33 @@ func ckzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
 
 	return blobs, nil
 }
+
+// ckzgRecoverCells recovers all cells for each blob from a sufficient subset via
+// KZG erasure recovery. Unlike ckzgRecoverBlobs it keeps the full extended cell
+// set rather than reducing it to blobs.
+func ckzgRecoverCells(cells []Cell, cellIndices []uint64) ([]Cell, error) {
+	ckzgIniter.Do(ckzgInit)
+
+	blobCount := len(cells) / len(cellIndices)
+	out := make([]Cell, 0, blobCount*CellsPerBlob)
+
+	offset := 0
+	for range blobCount {
+		kzgcells := make([]ckzg4844.Cell, 0, len(cellIndices))
+		for _, cell := range cells[offset : offset+len(cellIndices)] {
+			kzgcells = append(kzgcells, ckzg4844.Cell(cell))
+		}
+
+		extCells, err := ckzg4844.RecoverCells(cellIndices, kzgcells)
+		if err != nil {
+			return nil, err
+		}
+		for _, cell := range extCells {
+			out = append(out, Cell(cell))
+		}
+
+		offset = offset + len(cellIndices)
+	}
+
+	return out, nil
+}

crypto/kzg4844/kzg4844_ckzg_nocgo.go

@@ -85,3 +85,7 @@ func ckzgComputeCells(blobs []Blob) ([]Cell, error) {
 func ckzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
 	panic("unsupported platform")
 }
+
+func ckzgRecoverCells(cells []Cell, cellIndices []uint64) ([]Cell, error) {
+	panic("unsupported platform")
+}

crypto/kzg4844/kzg4844_gokzg.go

@@ -230,3 +230,34 @@ func gokzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
 
 	return blobs, nil
 }
+
+// gokzgRecoverCells recovers all cells for each blob from a sufficient subset via
+// KZG erasure recovery. Unlike gokzgRecoverBlobs it keeps the full extended cell
+// set rather than reducing it to blobs.
+func gokzgRecoverCells(cells []Cell, cellIndices []uint64) ([]Cell, error) {
+	gokzgIniter.Do(gokzgInit)
+
+	blobCount := len(cells) / len(cellIndices)
+	out := make([]Cell, 0, blobCount*CellsPerBlob)
+
+	offset := 0
+	for range blobCount {
+		kzgcells := make([]*gokzg4844.Cell, 0, len(cellIndices))
+		for _, cell := range cells[offset : offset+len(cellIndices)] {
+			gc := gokzg4844.Cell(cell)
+			kzgcells = append(kzgcells, &gc)
+		}
+
+		extCells, err := context.RecoverCells(cellIndices, kzgcells, 2)
+		if err != nil {
+			return nil, err
+		}
+		for _, cell := range extCells {
+			out = append(out, Cell(*cell))
+		}
+
+		offset = offset + len(cellIndices)
+	}
+
+	return out, nil
+}

crypto/kzg4844/kzg4844_test.go

@@ -18,6 +18,8 @@ package kzg4844
 
 import (
 	"crypto/rand"
+	"encoding/binary"
+	"math/big"
 	mrand "math/rand"
 	"slices"
 	"testing"
@@ -435,6 +437,89 @@ func testRecoverBlobWithInsufficientCells(t *testing.T, ckzg bool) {
 	}
 }
 
+func TestCKZGRecoverCells(t *testing.T)  { testRecoverCells(t, true) }
+func TestGoKZGRecoverCells(t *testing.T) { testRecoverCells(t, false) }
+
+// testRecoverCells checks that RecoverCells reconstructs the complete 128-cell
+// set for each blob, via both the fast systematic path (data cells present) and
+// the erasure-recovery slow path (non-data subset), byte-identical to the
+// original cells.
+func testRecoverCells(t *testing.T, ckzg bool) {
+	defer switchBackend(t, ckzg)()
+
+	const blobCount = 2
+	d := newBlobs(t, blobCount)
+
+	collect := func(indices []uint64) []Cell {
+		var cs []Cell
+		for bi := range blobCount {
+			for _, idx := range indices {
+				cs = append(cs, d.cells[bi*CellsPerBlob+int(idx)])
+			}
+		}
+		return cs
+	}
+	seq := func(start, n int) []uint64 {
+		idx := make([]uint64, n)
+		for i := range idx {
+			idx[i] = uint64(start + i)
+		}
+		return idx
+	}
+	assertRecoversAll := func(name string, indices []uint64) {
+		t.Helper()
+		got, err := RecoverCells(collect(indices), indices)
+		if err != nil {
+			t.Fatalf("%s: RecoverCells failed: %v", name, err)
+		}
+		if len(got) != blobCount*CellsPerBlob {
+			t.Fatalf("%s: got %d cells, want %d", name, len(got), blobCount*CellsPerBlob)
+		}
+		for i := range d.cells {
+			if got[i] != d.cells[i] {
+				t.Fatalf("%s: cell %d does not match original", name, i)
+			}
+		}
+	}
+
+	// Fast path: exactly the data cells.
+	assertRecoversAll("fast/data-0..63", seq(0, DataPerBlob))
+	// Slow path: a non-data 64-cell subset (indices 32..95).
+	assertRecoversAll("slow/non-data-32..95", seq(32, DataPerBlob))
+	// Full custody: all cells present (still takes the fast path).
+	assertRecoversAll("full-0..127", seq(0, CellsPerBlob))
+
+	// Fewer than DataPerBlob cells cannot be recovered.
+	short := seq(0, DataPerBlob-1)
+	if _, err := RecoverCells(collect(short), short); err == nil {
+		t.Fatalf("expected error with only %d cells", DataPerBlob-1)
+	}
+
+	// Malformed indices (duplicate tail): the fast path declines and the
+	// erasure path rejects, so both agree on refusal.
+	dup := append(seq(0, DataPerBlob), DataPerBlob-1)
+	if _, err := RecoverCells(collect(dup), dup); err == nil {
+		t.Fatalf("expected error for duplicate index")
+	}
+
+	// Randomized subsets exercise the erasure path with arbitrary shapes
+	// (kept to a few iterations: each is a full erasure decode).
+	for iter := range 3 {
+		rng := mrand.New(mrand.NewSource(int64(iter)))
+		n := DataPerBlob + rng.Intn(CellsPerBlob-DataPerBlob)
+		assertRecoversAll("random", randCellIndices(rng, n))
+	}
+}
+
+// erasureRecoverBlobs runs the KZG erasure recovery directly, bypassing the
+// concat fast path RecoverBlobs takes, so the two can be compared.
+func erasureRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
+	if useCKZG.Load() {
+		return ckzgRecoverBlobs(cells, cellIndices)
+	}
+	return gokzgRecoverBlobs(cells, cellIndices)
+}
+
 func TestCKZGBlobsFromDataCells(t *testing.T)  { testBlobsFromDataCells(t, true) }
 func TestGoKZGBlobsFromDataCells(t *testing.T) { testBlobsFromDataCells(t, false) }
 
@@ -444,7 +529,7 @@ func TestGoKZGBlobsFromDataCells(t *testing.T) { testBlobsFromDataCells(t, false
 func testBlobsFromDataCells(t *testing.T, ckzg bool) {
 	defer switchBackend(t, ckzg)()
 
-	const blobCount = 3
+	const blobCount = 2
 	d := newBlobs(t, blobCount)
 
 	// collect gathers the cells for the given per-blob indices across all blobs.
@@ -458,7 +543,7 @@ func testBlobsFromDataCells(t *testing.T, ckzg bool) {
 		return cells
 	}
 	// assertRecovers checks the fast path succeeds and matches both the original
-	// blobs and RecoverBlobs.
+	// blobs and the erasure recovery it stands in for.
 	assertRecovers := func(name string, indices []uint64) {
 		t.Helper()
 		cells := collect(indices)
@@ -466,16 +551,16 @@ func testBlobsFromDataCells(t *testing.T, ckzg bool) {
 		if !ok {
 			t.Fatalf("%s: fast path declined, expected success", name)
 		}
-		slow, err := RecoverBlobs(cells, indices)
+		slow, err := erasureRecoverBlobs(cells, indices)
 		if err != nil {
-			t.Fatalf("%s: RecoverBlobs failed: %v", name, err)
+			t.Fatalf("%s: erasure recovery failed: %v", name, err)
 		}
 		for i := range d.blobs {
 			if fast[i] != d.blobs[i] {
 				t.Fatalf("%s: fast blob %d does not match original", name, i)
 			}
 			if fast[i] != slow[i] {
-				t.Fatalf("%s: fast blob %d does not match RecoverBlobs", name, i)
+				t.Fatalf("%s: fast blob %d does not match the erasure recovery", name, i)
 			}
 		}
 	}
@@ -538,6 +623,38 @@ func testBlobsFromDataCells(t *testing.T, ckzg bool) {
 		t.Fatalf("out-of-range-tail: fast path succeeded, expected decline")
 	}
 
+	// Non-canonical field elements: the fast path bypasses the KZG library, so
+	// it has to reject what that library would reject while deserializing. The
+	// offending element goes in the last slot of the last cell of the last blob,
+	// so a check that only looked at the first element, cell or blob would still
+	// be caught, and it is the modulus itself, the tightest non-canonical value.
+	var modulus [32]byte
+	fr.Modulus().FillBytes(modulus[:])
+	poison := func(cells []Cell) {
+		last := &cells[len(cells)-1]
+		copy(last[len(last)-32:], modulus[:])
+	}
+	// In a data cell, which the concatenation reads:
+	badData := slices.Clone(collect(dataIndices))
+	poison(badData)
+	if _, ok := blobsFromDataCells(badData, dataIndices); ok {
+		t.Fatalf("non-canonical-data: fast path succeeded, expected decline")
+	}
+	if _, err := RecoverBlobs(badData, dataIndices); err == nil {
+		t.Fatalf("non-canonical-data: RecoverBlobs succeeded, expected error")
+	}
+	// And in a tail cell, which it ignores: declining keeps RecoverBlobs
+	// rejecting exactly what the erasure path rejects.
+	withTail := append(slices.Clone(dataIndices), DataPerBlob)
+	badTail := collect(withTail)
+	poison(badTail)
+	if _, ok := blobsFromDataCells(badTail, withTail); ok {
+		t.Fatalf("non-canonical-tail: fast path succeeded, expected decline")
+	}
+	if _, err := RecoverBlobs(badTail, withTail); err == nil {
+		t.Fatalf("non-canonical-tail: RecoverBlobs succeeded, expected error")
+	}
+
 	// Single blob: the slicing math must hold for blobCount == 1 too.
 	d1 := newBlobs(t, 1)
 	single, ok := blobsFromDataCells(d1.cells[:DataPerBlob], dataIndices)
@@ -562,16 +679,16 @@ func testBlobsFromDataCells(t *testing.T, ckzg bool) {
 	}
 }
 
-func TestCKZGRecoverBlobsUnchecked(t *testing.T)  { testRecoverBlobsUnchecked(t, true) }
-func TestGoKZGRecoverBlobsUnchecked(t *testing.T) { testRecoverBlobsUnchecked(t, false) }
+func TestCKZGRecoverBlobsFastPath(t *testing.T)  { testRecoverBlobsFastPath(t, true) }
+func TestGoKZGRecoverBlobsFastPath(t *testing.T) { testRecoverBlobsFastPath(t, false) }
 
-// testRecoverBlobsUnchecked checks that the unchecked recovery takes the
-// KZG-free fast path when the data cells are present and falls back to full
-// erasure recovery otherwise, matching the original blobs in both cases.
-func testRecoverBlobsUnchecked(t *testing.T, ckzg bool) {
+// testRecoverBlobsFastPath checks that RecoverBlobs takes the KZG-free fast
+// path when the data cells are present and falls back to full erasure recovery
+// otherwise, matching the original blobs in both cases.
+func testRecoverBlobsFastPath(t *testing.T, ckzg bool) {
 	defer switchBackend(t, ckzg)()
 
-	const blobCount = 3
+	const blobCount = 2
 	d := newBlobs(t, blobCount)
 
 	// collect gathers the cells for the given per-blob indices across all blobs.
@@ -588,7 +705,7 @@ func testRecoverBlobsUnchecked(t *testing.T, ckzg bool) {
 	// and matches the original blobs.
 	assertRecovers := func(name string, indices []uint64) {
 		t.Helper()
-		blobs, err := RecoverBlobsUnchecked(collect(indices), indices)
+		blobs, err := RecoverBlobs(collect(indices), indices)
 		if err != nil {
 			t.Fatalf("%s: recovery failed: %v", name, err)
 		}
@@ -619,9 +736,95 @@ func testRecoverBlobsUnchecked(t *testing.T, ckzg bool) {
 	}
 	assertRecovers("sparse (fallback)", sparse)
 
-	// Insufficient cells: recovery must error, like RecoverBlobs.
+	// Insufficient cells: recovery must error on either path.
 	short := dataIndices[:DataPerBlob-1]
-	if _, err := RecoverBlobsUnchecked(collect(short), short); err == nil {
+	if _, err := RecoverBlobs(collect(short), short); err == nil {
 		t.Fatalf("insufficient: expected error, got none")
 	}
+
+	// A redundant cell that is canonical but inconsistent with the data pins the
+	// one intentional divergence between the paths: the data cells decide, so the
+	// blobs come back correct, where the erasure recovery would have mixed the
+	// conflicting cell into the polynomial and returned neither faithfully.
+	conflicting := append(slices.Clone(dataIndices), DataPerBlob)
+	cells := collect(conflicting)
+	clear(cells[DataPerBlob][:]) // zero is canonical, and is not the real cell
+	if !isCanonicalCell(&cells[DataPerBlob]) {
+		t.Fatalf("conflicting-tail: test setup must leave the cell canonical")
+	}
+	blobs, err := RecoverBlobs(cells, conflicting)
+	if err != nil {
+		t.Fatalf("conflicting-tail: RecoverBlobs failed: %v", err)
+	}
+	for i := range d.blobs {
+		if blobs[i] != d.blobs[i] {
+			t.Fatalf("conflicting-tail: blob %d was not taken from the data cells", i)
+		}
+	}
+}
+
+// TestFieldModulusLimbs pins the hardcoded modulus limbs used by the
+// canonicalness check against the field's own definition.
+func TestFieldModulusLimbs(t *testing.T) {
+	var want [32]byte
+	fr.Modulus().FillBytes(want[:])
+
+	var got [32]byte
+	binary.BigEndian.PutUint64(got[0:8], frModulusW0)
+	binary.BigEndian.PutUint64(got[8:16], frModulusW1)
+	binary.BigEndian.PutUint64(got[16:24], frModulusW2)
+	binary.BigEndian.PutUint64(got[24:32], frModulusW3)
+
+	if got != want {
+		t.Fatalf("modulus limbs encode %x, field modulus is %x", got, want)
+	}
+}
+
+// TestIsCanonicalFieldElement cross-checks the hand-rolled comparison against
+// the field implementation whose deserialization it stands in for: the two
+// extremes, uniform random inputs, and the boundary region where the limb
+// comparison chain has to walk past equal limbs.
+func TestIsCanonicalFieldElement(t *testing.T) {
+	var (
+		e   fr.Element
+		buf [32]byte
+	)
+	check := func() {
+		t.Helper()
+		want := e.SetBytesCanonical(buf[:]) == nil
+		if got := isCanonicalFieldElement(buf[:]); got != want {
+			t.Fatalf("isCanonicalFieldElement(%x) = %v, library says %v", buf, got, want)
+		}
+	}
+	// The extremes, which random input never produces: all-zero (buf as it
+	// stands) and all-ones.
+	check()
+	for i := range buf {
+		buf[i] = 0xff
+	}
+	check()
+
+	// Uniform 32-byte values: a little under half are canonical, so both
+	// answers get exercised.
+	rng := mrand.New(mrand.NewSource(1))
+	for range 4096 {
+		if _, err := rng.Read(buf[:]); err != nil {
+			t.Fatal(err)
+		}
+		check()
+	}
+	// One step either side of each limb's modulus value: only these inputs
+	// reach the comparison of the limb in question.
+	mod := fr.Modulus()
+	for limb := range 4 {
+		unit := new(big.Int).Lsh(big.NewInt(1), uint(64*(3-limb)))
+		for _, delta := range []int64{-2, -1, 0, 1, 2} {
+			v := new(big.Int).Add(mod, new(big.Int).Mul(big.NewInt(delta), unit))
+			if v.Sign() < 0 || v.BitLen() > 256 {
+				continue
+			}
+			v.FillBytes(buf[:])
+			check()
+		}
+	}
 }