diff --git a/vendor/github.com/dchest/siphash/README.md b/vendor/github.com/dchest/siphash/README.md new file mode 100644 index 0000000000..a3645e191f --- /dev/null +++ b/vendor/github.com/dchest/siphash/README.md @@ -0,0 +1,69 @@ +SipHash (Go) +============ + +[![Build Status](https://travis-ci.org/dchest/siphash.svg)](https://travis-ci.org/dchest/siphash) + +Go implementation of SipHash-2-4, a fast short-input PRF created by +Jean-Philippe Aumasson and Daniel J. Bernstein (http://131002.net/siphash/). + + +## Installation + + $ go get github.com/dchest/siphash + +## Usage + + import "github.com/dchest/siphash" + +There are two ways to use this package. +The slower one is to use the standard hash.Hash64 interface: + + h := siphash.New(key) + h.Write([]byte("Hello")) + sum := h.Sum(nil) // returns 8-byte []byte + +or + + sum64 := h.Sum64() // returns uint64 + +The faster one is to use Hash() function, which takes two uint64 parts of +16-byte key and a byte slice, and returns uint64 hash: + + sum64 := siphash.Hash(key0, key1, []byte("Hello")) + +The keys and output are little-endian. + + +## Functions + +### func Hash(k0, k1 uint64, p []byte) uint64 + +Hash returns the 64-bit SipHash-2-4 of the given byte slice with two +64-bit parts of 128-bit key: k0 and k1. + +### func Hash128(k0, k1 uint64, p []byte) (uint64, uint64) + +Hash128 returns the 128-bit SipHash-2-4 of the given byte slice with two +64-bit parts of 128-bit key: k0 and k1. + +Note that 128-bit SipHash is considered experimental by SipHash authors at this time. + +### func New(key []byte) hash.Hash64 + +New returns a new hash.Hash64 computing SipHash-2-4 with 16-byte key. + +### func New128(key []byte) hash.Hash + +New128 returns a new hash.Hash computing SipHash-2-4 with 16-byte key and 16-byte output. + +Note that 16-byte output is considered experimental by SipHash authors at this time. + + +## Public domain dedication + +Written by Dmitry Chestnykh and Damian Gryski. + +To the extent possible under law, the authors have dedicated all copyright +and related and neighboring rights to this software to the public domain +worldwide. This software is distributed without any warranty. +http://creativecommons.org/publicdomain/zero/1.0/ diff --git a/vendor/github.com/dchest/siphash/blocks.go b/vendor/github.com/dchest/siphash/blocks.go new file mode 100644 index 0000000000..817b215e37 --- /dev/null +++ b/vendor/github.com/dchest/siphash/blocks.go @@ -0,0 +1,148 @@ +// +build !arm,!amd64 appengine gccgo + +package siphash + +func once(d *digest) { + blocks(d, d.x[:]) +} + +func finalize(d *digest) uint64 { + d0 := *d + once(&d0) + + v0, v1, v2, v3 := d0.v0, d0.v1, d0.v2, d0.v3 + v2 ^= 0xff + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 3. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 4. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + return v0 ^ v1 ^ v2 ^ v3 +} + +func blocks(d *digest, p []uint8) { + v0, v1, v2, v3 := d.v0, d.v1, d.v2, d.v3 + + for len(p) >= BlockSize { + m := uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | + uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 + + v3 ^= m + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + v0 ^= m + + p = p[BlockSize:] + } + + d.v0, d.v1, d.v2, d.v3 = v0, v1, v2, v3 +} diff --git a/vendor/github.com/dchest/siphash/blocks_amd64.s b/vendor/github.com/dchest/siphash/blocks_amd64.s new file mode 100644 index 0000000000..2d4816f397 --- /dev/null +++ b/vendor/github.com/dchest/siphash/blocks_amd64.s @@ -0,0 +1,86 @@ +// +build amd64,!appengine,!gccgo + +#define ROUND(v0, v1, v2, v3) \ + ADDQ v1, v0; \ + RORQ $51, v1; \ + ADDQ v3, v2; \ + XORQ v0, v1; \ + RORQ $48, v3; \ + RORQ $32, v0; \ + XORQ v2, v3; \ + ADDQ v1, v2; \ + ADDQ v3, v0; \ + RORQ $43, v3; \ + RORQ $47, v1; \ + XORQ v0, v3; \ + XORQ v2, v1; \ + RORQ $32, v2 + +// blocks(d *digest, data []uint8) +TEXT ·blocks(SB),4,$0-32 + MOVQ d+0(FP), BX + MOVQ 0(BX), R9 // R9 = v0 + MOVQ 8(BX), R10 // R10 = v1 + MOVQ 16(BX), R11 // R11 = v2 + MOVQ 24(BX), R12 // R12 = v3 + MOVQ p_base+8(FP), DI // DI = *uint64 + MOVQ p_len+16(FP), SI // SI = nblocks + XORL DX, DX // DX = index (0) + SHRQ $3, SI // SI /= 8 +body: + CMPQ DX, SI + JGE end + MOVQ 0(DI)(DX*8), CX // CX = m + XORQ CX, R12 + ROUND(R9, R10, R11, R12) + ROUND(R9, R10, R11, R12) + XORQ CX, R9 + ADDQ $1, DX + JMP body +end: + MOVQ R9, 0(BX) + MOVQ R10, 8(BX) + MOVQ R11, 16(BX) + MOVQ R12, 24(BX) + RET + +// once(d *digest) +TEXT ·once(SB),4,$0-8 + MOVQ d+0(FP), BX + MOVQ 0(BX), R9 // R9 = v0 + MOVQ 8(BX), R10 // R10 = v1 + MOVQ 16(BX), R11 // R11 = v2 + MOVQ 24(BX), R12 // R12 = v3 + MOVQ 48(BX), CX // CX = d.x[:] + XORQ CX, R12 + ROUND(R9, R10, R11, R12) + ROUND(R9, R10, R11, R12) + XORQ CX, R9 + MOVQ R9, 0(BX) + MOVQ R10, 8(BX) + MOVQ R11, 16(BX) + MOVQ R12, 24(BX) + RET + +// finalize(d *digest) uint64 +TEXT ·finalize(SB),4,$0-16 + MOVQ d+0(FP), BX + MOVQ 0(BX), R9 // R9 = v0 + MOVQ 8(BX), R10 // R10 = v1 + MOVQ 16(BX), R11 // R11 = v2 + MOVQ 24(BX), R12 // R12 = v3 + MOVQ 48(BX), CX // CX = d.x[:] + XORQ CX, R12 + ROUND(R9, R10, R11, R12) + ROUND(R9, R10, R11, R12) + XORQ CX, R9 + NOTB R11 + ROUND(R9, R10, R11, R12) + ROUND(R9, R10, R11, R12) + ROUND(R9, R10, R11, R12) + ROUND(R9, R10, R11, R12) + XORQ R12, R11 + XORQ R10, R9 + XORQ R11, R9 + MOVQ R9, ret+8(FP) + RET diff --git a/vendor/github.com/dchest/siphash/blocks_arm.s b/vendor/github.com/dchest/siphash/blocks_arm.s new file mode 100644 index 0000000000..42dcd23d59 --- /dev/null +++ b/vendor/github.com/dchest/siphash/blocks_arm.s @@ -0,0 +1,144 @@ +#include "textflag.h" +#define R10 g +#define ROUND()\ + ADD.S R2,R0,R0;\ + ADC R3,R1,R1;\ + EOR R2<<13,R0,R8;\ + EOR R3>>19,R8,R8;\ + EOR R2>>19,R1,R11;\ + EOR R3<<13,R11,R11;\ + ADD.S R6,R4,R4;\ + ADC R7,R5,R5;\ + EOR R6<<16,R4,R2;\ + EOR R7>>16,R2,R2;\ + EOR R6>>16,R5,R3;\ + EOR R7<<16,R3,R3;\ + ADD.S R2,R1,R1;\ + ADC R3,R0,R0;\ + EOR R2<<21,R1,R6;\ + EOR R3>>11,R6,R6;\ + EOR R2>>11,R0,R7;\ + EOR R3<<21,R7,R7;\ + ADD.S R8,R4,R4;\ + ADC R11,R5,R5;\ + EOR R8<<17,R4,R2;\ + EOR R11>>15,R2,R2;\ + EOR R8>>15,R5,R3;\ + EOR R11<<17,R3,R3;\ + ADD.S R2,R1,R1;\ + ADC R3,R0,R0;\ + EOR R2<<13,R1,R8;\ + EOR R3>>19,R8,R8;\ + EOR R2>>19,R0,R11;\ + EOR R3<<13,R11,R11;\ + ADD.S R6,R5,R5;\ + ADC R7,R4,R4;\ + EOR R6<<16,R5,R2;\ + EOR R7>>16,R2,R2;\ + EOR R6>>16,R4,R3;\ + EOR R7<<16,R3,R3;\ + ADD.S R2,R0,R0;\ + ADC R3,R1,R1;\ + EOR R2<<21,R0,R6;\ + EOR R3>>11,R6,R6;\ + EOR R2>>11,R1,R7;\ + EOR R3<<21,R7,R7;\ + ADD.S R8,R5,R5;\ + ADC R11,R4,R4;\ + EOR R8<<17,R5,R2;\ + EOR R11>>15,R2,R2;\ + EOR R8>>15,R4,R3;\ + EOR R11<<17,R3,R3;\ + +// once(d *digest) +TEXT ·once(SB),NOSPLIT,$4-4 + MOVW d+0(FP),R8 + MOVM.IA (R8),[R0,R1,R2,R3,R4,R5,R6,R7] + MOVW 48(R8),R12 + MOVW 52(R8),R14 + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + MOVW d+0(FP),R8 + MOVM.IA [R0,R1,R2,R3,R4,R5,R6,R7],(R8) + RET + +// finalize(d *digest) uint64 +TEXT ·finalize(SB),NOSPLIT,$4-12 + MOVW d+0(FP),R8 + MOVM.IA (R8),[R0,R1,R2,R3,R4,R5,R6,R7] + MOVW 48(R8),R12 + MOVW 52(R8),R14 + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + EOR $255,R4 + ROUND() + ROUND() + EOR R2,R0,R0 + EOR R3,R1,R1 + EOR R6,R4,R4 + EOR R7,R5,R5 + EOR R4,R0,R0 + EOR R5,R1,R1 + MOVW R0,ret_lo+4(FP) + MOVW R1,ret_hi+8(FP) + RET + +// blocks(d *digest, data []uint8) +TEXT ·blocks(SB),NOSPLIT,$8-16 + MOVW R10,sav-8(SP) + MOVW d+0(FP),R8 + MOVM.IA (R8),[R0,R1,R2,R3,R4,R5,R6,R7] + MOVW p+4(FP),R10 + MOVW p_len+8(FP),R11 + ADD R10,R11,R11 + MOVW R11,endp-4(SP) + AND.S $3,R10,R8 + BNE blocksunaligned +blocksloop: + MOVM.IA.W (R10),[R12,R14] + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + MOVW endp-4(SP),R11 + CMP R11,R10 + BLO blocksloop + MOVW d+0(FP),R8 + MOVM.IA [R0,R1,R2,R3,R4,R5,R6,R7],(R8) + MOVW sav-8(SP),R10 + RET +blocksunaligned: + MOVB (R10),R12 + MOVB 1(R10),R11 + ORR R11<<8,R12,R12 + MOVB 2(R10),R11 + ORR R11<<16,R12,R12 + MOVB 3(R10),R11 + ORR R11<<24,R12,R12 + MOVB 4(R10),R14 + MOVB 5(R10),R11 + ORR R11<<8,R14,R14 + MOVB 6(R10),R11 + ORR R11<<16,R14,R14 + MOVB 7(R10),R11 + ORR R11<<24,R14,R14 + ADD $8,R10,R10 + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + MOVW endp-4(SP),R11 + CMP R11,R10 + BLO blocksunaligned + MOVW d+0(FP),R8 + MOVM.IA [R0,R1,R2,R3,R4,R5,R6,R7],(R8) + MOVW sav-8(SP),R10 + RET diff --git a/vendor/github.com/dchest/siphash/hash.go b/vendor/github.com/dchest/siphash/hash.go new file mode 100644 index 0000000000..0de28d0db5 --- /dev/null +++ b/vendor/github.com/dchest/siphash/hash.go @@ -0,0 +1,216 @@ +// +build !arm,!amd64 appengine gccgo + +// Written in 2012 by Dmitry Chestnykh. +// +// To the extent possible under law, the author have dedicated all copyright +// and related and neighboring rights to this software to the public domain +// worldwide. This software is distributed without any warranty. +// http://creativecommons.org/publicdomain/zero/1.0/ + +package siphash + +// Hash returns the 64-bit SipHash-2-4 of the given byte slice with two 64-bit +// parts of 128-bit key: k0 and k1. +func Hash(k0, k1 uint64, p []byte) uint64 { + // Initialization. + v0 := k0 ^ 0x736f6d6570736575 + v1 := k1 ^ 0x646f72616e646f6d + v2 := k0 ^ 0x6c7967656e657261 + v3 := k1 ^ 0x7465646279746573 + t := uint64(len(p)) << 56 + + // Compression. + for len(p) >= BlockSize { + m := uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | + uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 + v3 ^= m + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + v0 ^= m + p = p[BlockSize:] + } + + // Compress last block. + switch len(p) { + case 7: + t |= uint64(p[6]) << 48 + fallthrough + case 6: + t |= uint64(p[5]) << 40 + fallthrough + case 5: + t |= uint64(p[4]) << 32 + fallthrough + case 4: + t |= uint64(p[3]) << 24 + fallthrough + case 3: + t |= uint64(p[2]) << 16 + fallthrough + case 2: + t |= uint64(p[1]) << 8 + fallthrough + case 1: + t |= uint64(p[0]) + } + + v3 ^= t + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + v0 ^= t + + // Finalization. + v2 ^= 0xff + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 3. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 4. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + return v0 ^ v1 ^ v2 ^ v3 +} diff --git a/vendor/github.com/dchest/siphash/hash128.go b/vendor/github.com/dchest/siphash/hash128.go new file mode 100644 index 0000000000..634ce57cb6 --- /dev/null +++ b/vendor/github.com/dchest/siphash/hash128.go @@ -0,0 +1,302 @@ +// +build !arm,!amd64 appengine gccgo +// Written in 2012 by Dmitry Chestnykh. +// Modifications 2014 for 128-bit hash function by Damian Gryski. +// +// To the extent possible under law, the authors have dedicated all copyright +// and related and neighboring rights to this software to the public domain +// worldwide. This software is distributed without any warranty. +// http://creativecommons.org/publicdomain/zero/1.0/ + +package siphash + +// Hash returns the 128-bit SipHash-2-4 of the given byte slice with two 64-bit +// parts of 128-bit key: k0 and k1. +// +// Note that 128-bit SipHash is considered experimental by SipHash authors at this time. +func Hash128(k0, k1 uint64, p []byte) (uint64, uint64) { + // Initialization. + v0 := k0 ^ 0x736f6d6570736575 + v1 := k1 ^ 0x646f72616e646f6d + v2 := k0 ^ 0x6c7967656e657261 + v3 := k1 ^ 0x7465646279746573 + t := uint64(len(p)) << 56 + + v1 ^= 0xee + + // Compression. + for len(p) >= BlockSize { + m := uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | + uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 + v3 ^= m + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + v0 ^= m + p = p[BlockSize:] + } + + // Compress last block. + switch len(p) { + case 7: + t |= uint64(p[6]) << 48 + fallthrough + case 6: + t |= uint64(p[5]) << 40 + fallthrough + case 5: + t |= uint64(p[4]) << 32 + fallthrough + case 4: + t |= uint64(p[3]) << 24 + fallthrough + case 3: + t |= uint64(p[2]) << 16 + fallthrough + case 2: + t |= uint64(p[1]) << 8 + fallthrough + case 1: + t |= uint64(p[0]) + } + + v3 ^= t + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + v0 ^= t + + // Finalization. + v2 ^= 0xee + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 3. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 4. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + r0 := v0 ^ v1 ^ v2 ^ v3 + + v1 ^= 0xdd + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 3. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 4. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + r1 := v0 ^ v1 ^ v2 ^ v3 + + return r0, r1 +} diff --git a/vendor/github.com/dchest/siphash/hash128_amd64.s b/vendor/github.com/dchest/siphash/hash128_amd64.s new file mode 100644 index 0000000000..86605ccf9c --- /dev/null +++ b/vendor/github.com/dchest/siphash/hash128_amd64.s @@ -0,0 +1,292 @@ +// +build amd64,!appengine,!gccgo + +// This is a translation of the gcc output of FloodyBerry's pure-C public +// domain siphash implementation at https://github.com/floodyberry/siphash + +// This assembly code has been modified from the 64-bit output to the experiment 128-bit output. + +// SI = v0 +// AX = v1 +// CX = v2 +// DX = v3 + +// func Hash128(k0, k1 uint64, b []byte) (r0 uint64, r1 uint64) +TEXT ·Hash128(SB),4,$0-56 + MOVQ k0+0(FP),CX + MOVQ $0x736F6D6570736575,R9 + MOVQ k1+8(FP),DI + MOVQ $0x6C7967656E657261,BX + MOVQ $0x646F72616E646F6D,AX + MOVQ b_len+24(FP),DX + XORQ $0xEE,AX + MOVQ DX,R11 + MOVQ DX,R10 + XORQ CX,R9 + XORQ CX,BX + MOVQ $0x7465646279746573,CX + XORQ DI,AX + XORQ DI,CX + SHLQ $0x38,R11 + XORQ DI,DI + MOVQ b_base+16(FP),SI + ANDQ $0xFFFFFFFFFFFFFFF8,R10 + JE afterLoop + XCHGQ AX,AX +loopBody: + MOVQ 0(SI)(DI*1),R8 + ADDQ AX,R9 + RORQ $0x33,AX + XORQ R9,AX + RORQ $0x20,R9 + ADDQ $0x8,DI + XORQ R8,CX + ADDQ CX,BX + RORQ $0x30,CX + XORQ BX,CX + ADDQ AX,BX + RORQ $0x2F,AX + ADDQ CX,R9 + RORQ $0x2B,CX + XORQ BX,AX + XORQ R9,CX + RORQ $0x20,BX + ADDQ AX,R9 + ADDQ CX,BX + RORQ $0x33,AX + RORQ $0x30,CX + XORQ R9,AX + XORQ BX,CX + RORQ $0x20,R9 + ADDQ AX,BX + ADDQ CX,R9 + RORQ $0x2F,AX + RORQ $0x2B,CX + XORQ BX,AX + RORQ $0x20,BX + XORQ R9,CX + XORQ R8,R9 + CMPQ R10,DI + JA loopBody +afterLoop: + SUBQ R10,DX + + CMPQ DX,$0x7 + JA afterSwitch + + // no support for jump tables + + CMPQ DX,$0x7 + JE sw7 + + CMPQ DX,$0x6 + JE sw6 + + CMPQ DX,$0x5 + JE sw5 + + CMPQ DX,$0x4 + JE sw4 + + CMPQ DX,$0x3 + JE sw3 + + CMPQ DX,$0x2 + JE sw2 + + CMPQ DX,$0x1 + JE sw1 + + JMP afterSwitch + +sw7: MOVBQZX 6(SI)(DI*1),DX + SHLQ $0x30,DX + ORQ DX,R11 +sw6: MOVBQZX 0x5(SI)(DI*1),DX + SHLQ $0x28,DX + ORQ DX,R11 +sw5: MOVBQZX 0x4(SI)(DI*1),DX + SHLQ $0x20,DX + ORQ DX,R11 +sw4: MOVBQZX 0x3(SI)(DI*1),DX + SHLQ $0x18,DX + ORQ DX,R11 +sw3: MOVBQZX 0x2(SI)(DI*1),DX + SHLQ $0x10,DX + ORQ DX,R11 +sw2: MOVBQZX 0x1(SI)(DI*1),DX + SHLQ $0x8,DX + ORQ DX,R11 +sw1: MOVBQZX 0(SI)(DI*1),DX + ORQ DX,R11 +afterSwitch: + LEAQ (AX)(R9*1),SI + XORQ R11,CX + RORQ $0x33,AX + ADDQ CX,BX + MOVQ CX,DX + XORQ SI,AX + RORQ $0x30,DX + RORQ $0x20,SI + LEAQ 0(BX)(AX*1),CX + XORQ BX,DX + RORQ $0x2F,AX + ADDQ DX,SI + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + ADDQ AX,SI + RORQ $0x33,AX + ADDQ DX,CX + XORQ SI,AX + RORQ $0x30,DX + RORQ $0x20,SI + XORQ CX,DX + ADDQ AX,CX + RORQ $0x2F,AX + ADDQ DX,SI + XORQ CX,AX + RORQ $0x2B,DX + RORQ $0x20,CX + XORQ SI,DX + XORQ R11,SI + XORB $0xEE,CL + ADDQ AX,SI + RORQ $0x33,AX + ADDQ DX,CX + RORQ $0x30,DX + XORQ SI,AX + XORQ CX,DX + RORQ $0x20,SI + ADDQ AX,CX + ADDQ DX,SI + RORQ $0x2F,AX + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + ADDQ AX,SI + ADDQ DX,CX + RORQ $0x33,AX + RORQ $0x30,DX + XORQ SI,AX + RORQ $0x20,SI + XORQ CX,DX + ADDQ AX,CX + RORQ $0x2F,AX + ADDQ DX,SI + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + ADDQ AX,SI + ADDQ DX,CX + RORQ $0x33,AX + RORQ $0x30,DX + XORQ CX,DX + XORQ SI,AX + RORQ $0x20,SI + ADDQ DX,SI + ADDQ AX,CX + RORQ $0x2F,AX + XORQ CX,AX + RORQ $0x2B,DX + RORQ $0x20,CX + XORQ SI,DX + + // gcc optimized the tail end of this function differently. However, + // we need to preserve out registers to carry out the second stage of + // the finalization. This is a duplicate of an earlier finalization + // round. + + ADDQ AX,SI + RORQ $0x33,AX + ADDQ DX,CX + RORQ $0x30,DX + XORQ SI,AX + XORQ CX,DX + RORQ $0x20,SI + ADDQ AX,CX + ADDQ DX,SI + RORQ $0x2F,AX + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + + // Stuff the result into BX instead of AX as gcc had done + + MOVQ SI,BX + XORQ AX,BX + XORQ DX,BX + XORQ CX,BX + MOVQ BX,ret+40(FP) + + // Start the second finalization round + + XORB $0xDD,AL + ADDQ AX,SI + RORQ $0x33,AX + ADDQ DX,CX + RORQ $0x30,DX + XORQ SI,AX + XORQ CX,DX + RORQ $0x20,SI + ADDQ AX,CX + ADDQ DX,SI + RORQ $0x2F,AX + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + ADDQ AX,SI + ADDQ DX,CX + RORQ $0x33,AX + RORQ $0x30,DX + XORQ SI,AX + RORQ $0x20,SI + XORQ CX,DX + ADDQ AX,CX + RORQ $0x2F,AX + ADDQ DX,SI + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + ADDQ AX,SI + ADDQ DX,CX + RORQ $0x33,AX + RORQ $0x30,DX + XORQ CX,DX + XORQ SI,AX + RORQ $0x20,SI + ADDQ DX,SI + ADDQ AX,CX + RORQ $0x2F,AX + XORQ CX,AX + RORQ $0x2B,DX + RORQ $0x20,CX + XORQ SI,DX + + ADDQ AX,SI + RORQ $0x33,AX + ADDQ DX,CX + RORQ $0x30,DX + XORQ SI,AX + XORQ CX,DX + RORQ $0x20,SI + ADDQ AX,CX + ADDQ DX,SI + RORQ $0x2F,AX + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + + MOVQ SI,BX + XORQ AX,BX + XORQ DX,BX + XORQ CX,BX + MOVQ BX,ret1+48(FP) + + RET diff --git a/vendor/github.com/dchest/siphash/hash128_arm.s b/vendor/github.com/dchest/siphash/hash128_arm.s new file mode 100644 index 0000000000..eca35293d2 --- /dev/null +++ b/vendor/github.com/dchest/siphash/hash128_arm.s @@ -0,0 +1,169 @@ +#include "textflag.h" +#define R10 g +#define ROUND()\ + ADD.S R2,R0,R0;\ + ADC R3,R1,R1;\ + EOR R2<<13,R0,R8;\ + EOR R3>>19,R8,R8;\ + EOR R2>>19,R1,R11;\ + EOR R3<<13,R11,R11;\ + ADD.S R6,R4,R4;\ + ADC R7,R5,R5;\ + EOR R6<<16,R4,R2;\ + EOR R7>>16,R2,R2;\ + EOR R6>>16,R5,R3;\ + EOR R7<<16,R3,R3;\ + ADD.S R2,R1,R1;\ + ADC R3,R0,R0;\ + EOR R2<<21,R1,R6;\ + EOR R3>>11,R6,R6;\ + EOR R2>>11,R0,R7;\ + EOR R3<<21,R7,R7;\ + ADD.S R8,R4,R4;\ + ADC R11,R5,R5;\ + EOR R8<<17,R4,R2;\ + EOR R11>>15,R2,R2;\ + EOR R8>>15,R5,R3;\ + EOR R11<<17,R3,R3;\ + ADD.S R2,R1,R1;\ + ADC R3,R0,R0;\ + EOR R2<<13,R1,R8;\ + EOR R3>>19,R8,R8;\ + EOR R2>>19,R0,R11;\ + EOR R3<<13,R11,R11;\ + ADD.S R6,R5,R5;\ + ADC R7,R4,R4;\ + EOR R6<<16,R5,R2;\ + EOR R7>>16,R2,R2;\ + EOR R6>>16,R4,R3;\ + EOR R7<<16,R3,R3;\ + ADD.S R2,R0,R0;\ + ADC R3,R1,R1;\ + EOR R2<<21,R0,R6;\ + EOR R3>>11,R6,R6;\ + EOR R2>>11,R1,R7;\ + EOR R3<<21,R7,R7;\ + ADD.S R8,R5,R5;\ + ADC R11,R4,R4;\ + EOR R8<<17,R5,R2;\ + EOR R11>>15,R2,R2;\ + EOR R8>>15,R4,R3;\ + EOR R11<<17,R3,R3;\ + +// Hash128(k0, k1 uint64, b []byte) (uint64, uint64) +TEXT ·Hash128(SB),NOSPLIT,$8-44 + MOVW R10,sav-8(SP) + MOVW k0_lo+0(FP),R12 + MOVW k0_hi+4(FP),R14 + MOVW $0x70736575,R0 + MOVW $0x736f6d65,R1 + MOVW $0x6e657261,R4 + MOVW $0x6c796765,R5 + EOR R12,R0,R0 + EOR R14,R1,R1 + EOR R12,R4,R4 + EOR R14,R5,R5 + MOVW k1_lo+8(FP),R12 + MOVW k1_hi+12(FP),R14 + MOVW $0x6e646f83,R2 + MOVW $0x646f7261,R3 + MOVW $0x79746573,R6 + MOVW $0x74656462,R7 + EOR R12,R2,R2 + EOR R14,R3,R3 + EOR R12,R6,R6 + EOR R14,R7,R7 + MOVW b+16(FP),R10 + MOVW b_len+20(FP),R11 + ADD R10,R11,R11 + MOVW R11,endb-4(SP) +hashloop128: + MOVW endb-4(SP),R11 + SUB R10,R11,R11 + SUB.S $8,R11 + BLO hashend128 + MOVM.IA.W (R10),[R12,R14] + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + B hashloop128 +hashloop128unaligned: + MOVW endb-4(SP),R11 + SUB R10,R11,R11 + SUB.S $8,R11 + BLO hashend128 + MOVB (R10),R12 + MOVB 1(R10),R11 + ORR R11<<8,R12,R12 + MOVB 2(R10),R11 + ORR R11<<16,R12,R12 + MOVB 3(R10),R11 + ORR R11<<24,R12,R12 + MOVB 4(R10),R14 + MOVB 5(R10),R11 + ORR R11<<8,R14,R14 + MOVB 6(R10),R11 + ORR R11<<16,R14,R14 + MOVB 7(R10),R11 + ORR R11<<24,R14,R14 + ADD $8,R10,R10 + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + B hashloop128unaligned +hashend128: + MOVW $0x0,R12 + MOVW $0x0,R14 + RSB $0,R11,R11 + AND.S $7,R11 + BEQ hashlast128 + MOVW (R10),R12 + SLL $3,R11 + AND $63,R11 + SUB.S $32,R11,R11 + BEQ hashlast128 + BLO hashhi128 + MOVW R12<>R11,R12 + B hashlast128 +hashhi128: + ADD $32,R11 + MOVW 4(R10),R14 + MOVW R14<>R11,R14 +hashlast128: + MOVW b_len+20(FP),R11 + ORR R11<<24,R14,R14 + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + EOR $238,R4 + ROUND() + ROUND() + EOR R0,R2,R12 + EOR R1,R3,R14 + EOR R4,R12,R12 + EOR R5,R14,R14 + EOR R6,R12,R12 + EOR R7,R14,R14 + MOVW R12,ret_lo+28(FP) + MOVW R14,ret_hi+32(FP) + EOR $221,R2 + ROUND() + ROUND() + EOR R0,R2,R12 + EOR R1,R3,R14 + EOR R4,R12,R12 + EOR R5,R14,R14 + EOR R6,R12,R12 + EOR R7,R14,R14 + MOVW R12,unnamed_lo+36(FP) + MOVW R14,unnamed_hi+40(FP) + MOVW sav-8(SP),R10 + RET diff --git a/vendor/github.com/dchest/siphash/hash_amd64.s b/vendor/github.com/dchest/siphash/hash_amd64.s new file mode 100644 index 0000000000..0ca1631ab9 --- /dev/null +++ b/vendor/github.com/dchest/siphash/hash_amd64.s @@ -0,0 +1,201 @@ +// +build amd64,!appengine,!gccgo + +// This is a translation of the gcc output of FloodyBerry's pure-C public +// domain siphash implementation at https://github.com/floodyberry/siphash +// func Hash(k0, k1 uint64, b []byte) uint64 +TEXT ·Hash(SB),4,$0-48 + MOVQ k0+0(FP),CX + MOVQ $0x736F6D6570736575,R9 + MOVQ k1+8(FP),DI + MOVQ $0x6C7967656E657261,BX + MOVQ $0x646F72616E646F6D,AX + MOVQ b_len+24(FP),DX + MOVQ DX,R11 + MOVQ DX,R10 + XORQ CX,R9 + XORQ CX,BX + MOVQ $0x7465646279746573,CX + XORQ DI,AX + XORQ DI,CX + SHLQ $0x38,R11 + XORQ DI,DI + MOVQ b_base+16(FP),SI + ANDQ $0xFFFFFFFFFFFFFFF8,R10 + JE afterLoop + XCHGQ AX,AX +loopBody: + MOVQ 0(SI)(DI*1),R8 + ADDQ AX,R9 + RORQ $0x33,AX + XORQ R9,AX + RORQ $0x20,R9 + ADDQ $0x8,DI + XORQ R8,CX + ADDQ CX,BX + RORQ $0x30,CX + XORQ BX,CX + ADDQ AX,BX + RORQ $0x2F,AX + ADDQ CX,R9 + RORQ $0x2B,CX + XORQ BX,AX + XORQ R9,CX + RORQ $0x20,BX + ADDQ AX,R9 + ADDQ CX,BX + RORQ $0x33,AX + RORQ $0x30,CX + XORQ R9,AX + XORQ BX,CX + RORQ $0x20,R9 + ADDQ AX,BX + ADDQ CX,R9 + RORQ $0x2F,AX + RORQ $0x2B,CX + XORQ BX,AX + RORQ $0x20,BX + XORQ R9,CX + XORQ R8,R9 + CMPQ R10,DI + JA loopBody +afterLoop: + SUBQ R10,DX + + CMPQ DX,$0x7 + JA afterSwitch + + // no support for jump tables + + CMPQ DX,$0x7 + JE sw7 + + CMPQ DX,$0x6 + JE sw6 + + CMPQ DX,$0x5 + JE sw5 + + CMPQ DX,$0x4 + JE sw4 + + CMPQ DX,$0x3 + JE sw3 + + CMPQ DX,$0x2 + JE sw2 + + CMPQ DX,$0x1 + JE sw1 + + JMP afterSwitch + +sw7: MOVBQZX 6(SI)(DI*1),DX + SHLQ $0x30,DX + ORQ DX,R11 +sw6: MOVBQZX 0x5(SI)(DI*1),DX + SHLQ $0x28,DX + ORQ DX,R11 +sw5: MOVBQZX 0x4(SI)(DI*1),DX + SHLQ $0x20,DX + ORQ DX,R11 +sw4: MOVBQZX 0x3(SI)(DI*1),DX + SHLQ $0x18,DX + ORQ DX,R11 +sw3: MOVBQZX 0x2(SI)(DI*1),DX + SHLQ $0x10,DX + ORQ DX,R11 +sw2: MOVBQZX 0x1(SI)(DI*1),DX + SHLQ $0x8,DX + ORQ DX,R11 +sw1: MOVBQZX 0(SI)(DI*1),DX + ORQ DX,R11 +afterSwitch: + LEAQ (AX)(R9*1),SI + XORQ R11,CX + RORQ $0x33,AX + ADDQ CX,BX + MOVQ CX,DX + XORQ SI,AX + RORQ $0x30,DX + RORQ $0x20,SI + LEAQ 0(BX)(AX*1),CX + XORQ BX,DX + RORQ $0x2F,AX + ADDQ DX,SI + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + ADDQ AX,SI + RORQ $0x33,AX + ADDQ DX,CX + XORQ SI,AX + RORQ $0x30,DX + RORQ $0x20,SI + XORQ CX,DX + ADDQ AX,CX + RORQ $0x2F,AX + ADDQ DX,SI + XORQ CX,AX + RORQ $0x2B,DX + RORQ $0x20,CX + XORQ SI,DX + XORQ R11,SI + XORB $0xFF,CL + ADDQ AX,SI + RORQ $0x33,AX + ADDQ DX,CX + RORQ $0x30,DX + XORQ SI,AX + XORQ CX,DX + RORQ $0x20,SI + ADDQ AX,CX + ADDQ DX,SI + RORQ $0x2F,AX + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + ADDQ AX,SI + ADDQ DX,CX + RORQ $0x33,AX + RORQ $0x30,DX + XORQ SI,AX + RORQ $0x20,SI + XORQ CX,DX + ADDQ AX,CX + RORQ $0x2F,AX + ADDQ DX,SI + RORQ $0x2B,DX + XORQ CX,AX + XORQ SI,DX + RORQ $0x20,CX + ADDQ AX,SI + ADDQ DX,CX + RORQ $0x33,AX + RORQ $0x30,DX + XORQ CX,DX + XORQ SI,AX + RORQ $0x20,SI + ADDQ DX,SI + ADDQ AX,CX + RORQ $0x2F,AX + XORQ CX,AX + RORQ $0x2B,DX + RORQ $0x20,CX + XORQ SI,DX + ADDQ AX,SI + RORQ $0x33,AX + ADDQ DX,CX + XORQ SI,AX + RORQ $0x30,DX + XORQ CX,DX + ADDQ AX,CX + RORQ $0x2F,AX + XORQ CX,AX + RORQ $0x2B,DX + RORQ $0x20,CX + XORQ DX,AX + XORQ CX,AX + MOVQ AX,ret+40(FP) + RET diff --git a/vendor/github.com/dchest/siphash/hash_arm.s b/vendor/github.com/dchest/siphash/hash_arm.s new file mode 100644 index 0000000000..ddad8f87d7 --- /dev/null +++ b/vendor/github.com/dchest/siphash/hash_arm.s @@ -0,0 +1,160 @@ +#include "textflag.h" +#define R10 g +#define ROUND()\ + ADD.S R2,R0,R0;\ + ADC R3,R1,R1;\ + EOR R2<<13,R0,R8;\ + EOR R3>>19,R8,R8;\ + EOR R2>>19,R1,R11;\ + EOR R3<<13,R11,R11;\ + ADD.S R6,R4,R4;\ + ADC R7,R5,R5;\ + EOR R6<<16,R4,R2;\ + EOR R7>>16,R2,R2;\ + EOR R6>>16,R5,R3;\ + EOR R7<<16,R3,R3;\ + ADD.S R2,R1,R1;\ + ADC R3,R0,R0;\ + EOR R2<<21,R1,R6;\ + EOR R3>>11,R6,R6;\ + EOR R2>>11,R0,R7;\ + EOR R3<<21,R7,R7;\ + ADD.S R8,R4,R4;\ + ADC R11,R5,R5;\ + EOR R8<<17,R4,R2;\ + EOR R11>>15,R2,R2;\ + EOR R8>>15,R5,R3;\ + EOR R11<<17,R3,R3;\ + ADD.S R2,R1,R1;\ + ADC R3,R0,R0;\ + EOR R2<<13,R1,R8;\ + EOR R3>>19,R8,R8;\ + EOR R2>>19,R0,R11;\ + EOR R3<<13,R11,R11;\ + ADD.S R6,R5,R5;\ + ADC R7,R4,R4;\ + EOR R6<<16,R5,R2;\ + EOR R7>>16,R2,R2;\ + EOR R6>>16,R4,R3;\ + EOR R7<<16,R3,R3;\ + ADD.S R2,R0,R0;\ + ADC R3,R1,R1;\ + EOR R2<<21,R0,R6;\ + EOR R3>>11,R6,R6;\ + EOR R2>>11,R1,R7;\ + EOR R3<<21,R7,R7;\ + ADD.S R8,R5,R5;\ + ADC R11,R4,R4;\ + EOR R8<<17,R5,R2;\ + EOR R11>>15,R2,R2;\ + EOR R8>>15,R4,R3;\ + EOR R11<<17,R3,R3;\ + +// Hash(k0, k1 uint64, b []byte) uint64 +TEXT ·Hash(SB),NOSPLIT,$8-36 + MOVW R10,sav-8(SP) + MOVW k0_lo+0(FP),R12 + MOVW k0_hi+4(FP),R14 + MOVW $0x70736575,R0 + MOVW $0x736f6d65,R1 + MOVW $0x6e657261,R4 + MOVW $0x6c796765,R5 + EOR R12,R0,R0 + EOR R14,R1,R1 + EOR R12,R4,R4 + EOR R14,R5,R5 + MOVW k1_lo+8(FP),R12 + MOVW k1_hi+12(FP),R14 + MOVW $0x6e646f6d,R2 + MOVW $0x646f7261,R3 + MOVW $0x79746573,R6 + MOVW $0x74656462,R7 + EOR R12,R2,R2 + EOR R14,R3,R3 + EOR R12,R6,R6 + EOR R14,R7,R7 + MOVW b+16(FP),R10 + MOVW b_len+20(FP),R11 + ADD R10,R11,R11 + MOVW R11,endb-4(SP) + AND.S $3,R10,R8 + BNE hashloopunaligned +hashloop: + MOVW endb-4(SP),R11 + SUB R10,R11,R11 + SUB.S $8,R11 + BLO hashend + MOVM.IA.W (R10),[R12,R14] + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + B hashloop +hashloopunaligned: + MOVW endb-4(SP),R11 + SUB R10,R11,R11 + SUB.S $8,R11 + BLO hashend + MOVB (R10),R12 + MOVB 1(R10),R11 + ORR R11<<8,R12,R12 + MOVB 2(R10),R11 + ORR R11<<16,R12,R12 + MOVB 3(R10),R11 + ORR R11<<24,R12,R12 + MOVB 4(R10),R14 + MOVB 5(R10),R11 + ORR R11<<8,R14,R14 + MOVB 6(R10),R11 + ORR R11<<16,R14,R14 + MOVB 7(R10),R11 + ORR R11<<24,R14,R14 + ADD $8,R10,R10 + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + B hashloopunaligned +hashend: + MOVW $0x0,R12 + MOVW $0x0,R14 + RSB $0,R11,R11 + AND.S $7,R11 + BEQ hashlast + MOVW (R10),R12 + SLL $3,R11 + AND $63,R11 + SUB.S $32,R11,R11 + BEQ hashlast + BLO hashhi + MOVW R12<>R11,R12 + B hashlast +hashhi: + ADD $32,R11 + MOVW 4(R10),R14 + MOVW R14<>R11,R14 +hashlast: + MOVW b_len+20(FP),R11 + ORR R11<<24,R14,R14 + EOR R12,R6,R6 + EOR R14,R7,R7 + ROUND() + EOR R12,R0,R0 + EOR R14,R1,R1 + EOR $255,R4 + ROUND() + ROUND() + EOR R2,R0,R0 + EOR R3,R1,R1 + EOR R6,R4,R4 + EOR R7,R5,R5 + EOR R4,R0,R0 + EOR R5,R1,R1 + MOVW sav-8(SP),R10 + MOVW R0,ret_lo+28(FP) + MOVW R1,ret_hi+32(FP) + RET diff --git a/vendor/github.com/dchest/siphash/hash_asm.go b/vendor/github.com/dchest/siphash/hash_asm.go new file mode 100644 index 0000000000..13f540cb72 --- /dev/null +++ b/vendor/github.com/dchest/siphash/hash_asm.go @@ -0,0 +1,33 @@ +// +build arm amd64,!appengine,!gccgo + +// Written in 2012 by Dmitry Chestnykh. +// +// To the extent possible under law, the author have dedicated all copyright +// and related and neighboring rights to this software to the public domain +// worldwide. This software is distributed without any warranty. +// http://creativecommons.org/publicdomain/zero/1.0/ + +// This file contains a function definition for use with assembly implementations of Hash() + +package siphash + +//go:noescape + +// Hash returns the 64-bit SipHash-2-4 of the given byte slice with two 64-bit +// parts of 128-bit key: k0 and k1. +func Hash(k0, k1 uint64, b []byte) uint64 + +//go:noescape + +// Hash128 returns the 128-bit SipHash-2-4 of the given byte slice with two +// 64-bit parts of 128-bit key: k0 and k1. +func Hash128(k0, k1 uint64, b []byte) (uint64, uint64) + +//go:noescape +func blocks(d *digest, p []uint8) + +//go:noescape +func finalize(d *digest) uint64 + +//go:noescape +func once(d *digest) diff --git a/vendor/github.com/dchest/siphash/siphash.go b/vendor/github.com/dchest/siphash/siphash.go new file mode 100644 index 0000000000..4a3cb499a8 --- /dev/null +++ b/vendor/github.com/dchest/siphash/siphash.go @@ -0,0 +1,318 @@ +// Written in 2012-2014 by Dmitry Chestnykh. +// +// To the extent possible under law, the author have dedicated all copyright +// and related and neighboring rights to this software to the public domain +// worldwide. This software is distributed without any warranty. +// http://creativecommons.org/publicdomain/zero/1.0/ + +// Package siphash implements SipHash-2-4, a fast short-input PRF +// created by Jean-Philippe Aumasson and Daniel J. Bernstein. +package siphash + +import "hash" + +const ( + // BlockSize is the block size of hash algorithm in bytes. + BlockSize = 8 + + // Size is the size of hash output in bytes. + Size = 8 + + // Size128 is the size of 128-bit hash output in bytes. + Size128 = 16 +) + +type digest struct { + v0, v1, v2, v3 uint64 // state + k0, k1 uint64 // two parts of key + x [8]byte // buffer for unprocessed bytes + nx int // number of bytes in buffer x + size int // output size in bytes (8 or 16) + t uint8 // message bytes counter (mod 256) +} + +// newDigest returns a new digest with the given output size in bytes (must be 8 or 16). +func newDigest(size int, key []byte) *digest { + if size != Size && size != Size128 { + panic("size must be 8 or 16") + } + d := new(digest) + d.k0 = uint64(key[0]) | uint64(key[1])<<8 | uint64(key[2])<<16 | uint64(key[3])<<24 | + uint64(key[4])<<32 | uint64(key[5])<<40 | uint64(key[6])<<48 | uint64(key[7])<<56 + d.k1 = uint64(key[8]) | uint64(key[9])<<8 | uint64(key[10])<<16 | uint64(key[11])<<24 | + uint64(key[12])<<32 | uint64(key[13])<<40 | uint64(key[14])<<48 | uint64(key[15])<<56 + d.size = size + d.Reset() + return d +} + +// New returns a new hash.Hash64 computing SipHash-2-4 with 16-byte key and 8-byte output. +func New(key []byte) hash.Hash64 { + return newDigest(Size, key) +} + +// New128 returns a new hash.Hash computing SipHash-2-4 with 16-byte key and 16-byte output. +// +// Note that 16-byte output is considered experimental by SipHash authors at this time. +func New128(key []byte) hash.Hash { + return newDigest(Size128, key) +} + +func (d *digest) Reset() { + d.v0 = d.k0 ^ 0x736f6d6570736575 + d.v1 = d.k1 ^ 0x646f72616e646f6d + d.v2 = d.k0 ^ 0x6c7967656e657261 + d.v3 = d.k1 ^ 0x7465646279746573 + d.t = 0 + d.nx = 0 + if d.size == Size128 { + d.v1 ^= 0xee + } +} + +func (d *digest) Size() int { return d.size } + +func (d *digest) BlockSize() int { return BlockSize } + +func (d *digest) Write(p []byte) (nn int, err error) { + nn = len(p) + d.t += uint8(nn) + if d.nx > 0 { + n := len(p) + if n > BlockSize-d.nx { + n = BlockSize - d.nx + } + d.nx += copy(d.x[d.nx:], p) + if d.nx == BlockSize { + once(d) + d.nx = 0 + } + p = p[n:] + } + if len(p) >= BlockSize { + n := len(p) &^ (BlockSize - 1) + blocks(d, p[:n]) + p = p[n:] + } + if len(p) > 0 { + d.nx = copy(d.x[:], p) + } + return +} + +func (d *digest) Sum64() uint64 { + for i := d.nx; i < BlockSize-1; i++ { + d.x[i] = 0 + } + d.x[7] = d.t + return finalize(d) +} + +func (d0 *digest) sum128() (r0, r1 uint64) { + // Make a copy of d0 so that caller can keep writing and summing. + d := *d0 + + for i := d.nx; i < BlockSize-1; i++ { + d.x[i] = 0 + } + d.x[7] = d.t + blocks(&d, d.x[:]) + + v0, v1, v2, v3 := d.v0, d.v1, d.v2, d.v3 + v2 ^= 0xee + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 3. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 4. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + r0 = v0 ^ v1 ^ v2 ^ v3 + + v1 ^= 0xdd + + // Round 1. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 2. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 3. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + // Round 4. + v0 += v1 + v1 = v1<<13 | v1>>(64-13) + v1 ^= v0 + v0 = v0<<32 | v0>>(64-32) + + v2 += v3 + v3 = v3<<16 | v3>>(64-16) + v3 ^= v2 + + v0 += v3 + v3 = v3<<21 | v3>>(64-21) + v3 ^= v0 + + v2 += v1 + v1 = v1<<17 | v1>>(64-17) + v1 ^= v2 + v2 = v2<<32 | v2>>(64-32) + + r1 = v0 ^ v1 ^ v2 ^ v3 + + return r0, r1 +} + +func (d *digest) Sum(in []byte) []byte { + if d.size == Size { + r := d.Sum64() + in = append(in, + byte(r), + byte(r>>8), + byte(r>>16), + byte(r>>24), + byte(r>>32), + byte(r>>40), + byte(r>>48), + byte(r>>56)) + } else { + r0, r1 := d.sum128() + in = append(in, + byte(r0), + byte(r0>>8), + byte(r0>>16), + byte(r0>>24), + byte(r0>>32), + byte(r0>>40), + byte(r0>>48), + byte(r0>>56), + byte(r1), + byte(r1>>8), + byte(r1>>16), + byte(r1>>24), + byte(r1>>32), + byte(r1>>40), + byte(r1>>48), + byte(r1>>56)) + } + return in +} diff --git a/vendor/github.com/dchest/siphash/siphash_test.go b/vendor/github.com/dchest/siphash/siphash_test.go new file mode 100644 index 0000000000..b7617b8ba1 --- /dev/null +++ b/vendor/github.com/dchest/siphash/siphash_test.go @@ -0,0 +1,591 @@ +// Written in 2012 by Dmitry Chestnykh. +// +// To the extent possible under law, the author have dedicated all copyright +// and related and neighboring rights to this software to the public domain +// worldwide. This software is distributed without any warranty. +// http://creativecommons.org/publicdomain/zero/1.0/ + +package siphash + +import ( + "bytes" + "encoding/binary" + "testing" +) + +var zeroKey = make([]byte, 16) + +var golden = []struct { + k []byte + m []byte + r uint64 +}{ + { + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e}, + 0xa129ca6149be45e5, + }, + { + zeroKey, + []byte("Hello world"), + 0xc9e8a3021f3822d9, + }, + { + zeroKey, + []byte{}, // zero-length message + 0x1e924b9d737700d7, + }, + { + zeroKey, + []byte("12345678123"), + 0xf95d77ccdb0649f, + }, + { + zeroKey, + make([]byte, 8), + 0xe849e8bb6ffe2567, + }, + { + zeroKey, + make([]byte, 1535), + 0xe74d1c0ab64b2afa, + }, +} + +// Test vectors from reference implementation. +// +// SipHash-2-4 output with +// k = 00 01 02 ... +// and +// in = (empty string) +// in = 00 (1 byte) +// in = 00 01 (2 bytes) +// in = 00 01 02 (3 bytes) +// ... +// in = 00 01 02 ... 3e (63 bytes) +var goldenRef = [][]byte{ + {0x31, 0x0e, 0x0e, 0xdd, 0x47, 0xdb, 0x6f, 0x72}, + {0xfd, 0x67, 0xdc, 0x93, 0xc5, 0x39, 0xf8, 0x74}, + {0x5a, 0x4f, 0xa9, 0xd9, 0x09, 0x80, 0x6c, 0x0d}, + {0x2d, 0x7e, 0xfb, 0xd7, 0x96, 0x66, 0x67, 0x85}, + {0xb7, 0x87, 0x71, 0x27, 0xe0, 0x94, 0x27, 0xcf}, + {0x8d, 0xa6, 0x99, 0xcd, 0x64, 0x55, 0x76, 0x18}, + {0xce, 0xe3, 0xfe, 0x58, 0x6e, 0x46, 0xc9, 0xcb}, + {0x37, 0xd1, 0x01, 0x8b, 0xf5, 0x00, 0x02, 0xab}, + {0x62, 0x24, 0x93, 0x9a, 0x79, 0xf5, 0xf5, 0x93}, + {0xb0, 0xe4, 0xa9, 0x0b, 0xdf, 0x82, 0x00, 0x9e}, + {0xf3, 0xb9, 0xdd, 0x94, 0xc5, 0xbb, 0x5d, 0x7a}, + {0xa7, 0xad, 0x6b, 0x22, 0x46, 0x2f, 0xb3, 0xf4}, + {0xfb, 0xe5, 0x0e, 0x86, 0xbc, 0x8f, 0x1e, 0x75}, + {0x90, 0x3d, 0x84, 0xc0, 0x27, 0x56, 0xea, 0x14}, + {0xee, 0xf2, 0x7a, 0x8e, 0x90, 0xca, 0x23, 0xf7}, + {0xe5, 0x45, 0xbe, 0x49, 0x61, 0xca, 0x29, 0xa1}, + {0xdb, 0x9b, 0xc2, 0x57, 0x7f, 0xcc, 0x2a, 0x3f}, + {0x94, 0x47, 0xbe, 0x2c, 0xf5, 0xe9, 0x9a, 0x69}, + {0x9c, 0xd3, 0x8d, 0x96, 0xf0, 0xb3, 0xc1, 0x4b}, + {0xbd, 0x61, 0x79, 0xa7, 0x1d, 0xc9, 0x6d, 0xbb}, + {0x98, 0xee, 0xa2, 0x1a, 0xf2, 0x5c, 0xd6, 0xbe}, + {0xc7, 0x67, 0x3b, 0x2e, 0xb0, 0xcb, 0xf2, 0xd0}, + {0x88, 0x3e, 0xa3, 0xe3, 0x95, 0x67, 0x53, 0x93}, + {0xc8, 0xce, 0x5c, 0xcd, 0x8c, 0x03, 0x0c, 0xa8}, + {0x94, 0xaf, 0x49, 0xf6, 0xc6, 0x50, 0xad, 0xb8}, + {0xea, 0xb8, 0x85, 0x8a, 0xde, 0x92, 0xe1, 0xbc}, + {0xf3, 0x15, 0xbb, 0x5b, 0xb8, 0x35, 0xd8, 0x17}, + {0xad, 0xcf, 0x6b, 0x07, 0x63, 0x61, 0x2e, 0x2f}, + {0xa5, 0xc9, 0x1d, 0xa7, 0xac, 0xaa, 0x4d, 0xde}, + {0x71, 0x65, 0x95, 0x87, 0x66, 0x50, 0xa2, 0xa6}, + {0x28, 0xef, 0x49, 0x5c, 0x53, 0xa3, 0x87, 0xad}, + {0x42, 0xc3, 0x41, 0xd8, 0xfa, 0x92, 0xd8, 0x32}, + {0xce, 0x7c, 0xf2, 0x72, 0x2f, 0x51, 0x27, 0x71}, + {0xe3, 0x78, 0x59, 0xf9, 0x46, 0x23, 0xf3, 0xa7}, + {0x38, 0x12, 0x05, 0xbb, 0x1a, 0xb0, 0xe0, 0x12}, + {0xae, 0x97, 0xa1, 0x0f, 0xd4, 0x34, 0xe0, 0x15}, + {0xb4, 0xa3, 0x15, 0x08, 0xbe, 0xff, 0x4d, 0x31}, + {0x81, 0x39, 0x62, 0x29, 0xf0, 0x90, 0x79, 0x02}, + {0x4d, 0x0c, 0xf4, 0x9e, 0xe5, 0xd4, 0xdc, 0xca}, + {0x5c, 0x73, 0x33, 0x6a, 0x76, 0xd8, 0xbf, 0x9a}, + {0xd0, 0xa7, 0x04, 0x53, 0x6b, 0xa9, 0x3e, 0x0e}, + {0x92, 0x59, 0x58, 0xfc, 0xd6, 0x42, 0x0c, 0xad}, + {0xa9, 0x15, 0xc2, 0x9b, 0xc8, 0x06, 0x73, 0x18}, + {0x95, 0x2b, 0x79, 0xf3, 0xbc, 0x0a, 0xa6, 0xd4}, + {0xf2, 0x1d, 0xf2, 0xe4, 0x1d, 0x45, 0x35, 0xf9}, + {0x87, 0x57, 0x75, 0x19, 0x04, 0x8f, 0x53, 0xa9}, + {0x10, 0xa5, 0x6c, 0xf5, 0xdf, 0xcd, 0x9a, 0xdb}, + {0xeb, 0x75, 0x09, 0x5c, 0xcd, 0x98, 0x6c, 0xd0}, + {0x51, 0xa9, 0xcb, 0x9e, 0xcb, 0xa3, 0x12, 0xe6}, + {0x96, 0xaf, 0xad, 0xfc, 0x2c, 0xe6, 0x66, 0xc7}, + {0x72, 0xfe, 0x52, 0x97, 0x5a, 0x43, 0x64, 0xee}, + {0x5a, 0x16, 0x45, 0xb2, 0x76, 0xd5, 0x92, 0xa1}, + {0xb2, 0x74, 0xcb, 0x8e, 0xbf, 0x87, 0x87, 0x0a}, + {0x6f, 0x9b, 0xb4, 0x20, 0x3d, 0xe7, 0xb3, 0x81}, + {0xea, 0xec, 0xb2, 0xa3, 0x0b, 0x22, 0xa8, 0x7f}, + {0x99, 0x24, 0xa4, 0x3c, 0xc1, 0x31, 0x57, 0x24}, + {0xbd, 0x83, 0x8d, 0x3a, 0xaf, 0xbf, 0x8d, 0xb7}, + {0x0b, 0x1a, 0x2a, 0x32, 0x65, 0xd5, 0x1a, 0xea}, + {0x13, 0x50, 0x79, 0xa3, 0x23, 0x1c, 0xe6, 0x60}, + {0x93, 0x2b, 0x28, 0x46, 0xe4, 0xd7, 0x06, 0x66}, + {0xe1, 0x91, 0x5f, 0x5c, 0xb1, 0xec, 0xa4, 0x6c}, + {0xf3, 0x25, 0x96, 0x5c, 0xa1, 0x6d, 0x62, 0x9f}, + {0x57, 0x5f, 0xf2, 0x8e, 0x60, 0x38, 0x1b, 0xe5}, + {0x72, 0x45, 0x06, 0xeb, 0x4c, 0x32, 0x8a, 0x95}, +} + +var goldenRef128 = [][]byte{ + {0xa3, 0x81, 0x7f, 0x04, 0xba, 0x25, 0xa8, 0xe6, 0x6d, 0xf6, 0x72, 0x14, 0xc7, 0x55, 0x02, 0x93}, + {0xda, 0x87, 0xc1, 0xd8, 0x6b, 0x99, 0xaf, 0x44, 0x34, 0x76, 0x59, 0x11, 0x9b, 0x22, 0xfc, 0x45}, + {0x81, 0x77, 0x22, 0x8d, 0xa4, 0xa4, 0x5d, 0xc7, 0xfc, 0xa3, 0x8b, 0xde, 0xf6, 0x0a, 0xff, 0xe4}, + {0x9c, 0x70, 0xb6, 0x0c, 0x52, 0x67, 0xa9, 0x4e, 0x5f, 0x33, 0xb6, 0xb0, 0x29, 0x85, 0xed, 0x51}, + {0xf8, 0x81, 0x64, 0xc1, 0x2d, 0x9c, 0x8f, 0xaf, 0x7d, 0x0f, 0x6e, 0x7c, 0x7b, 0xcd, 0x55, 0x79}, + {0x13, 0x68, 0x87, 0x59, 0x80, 0x77, 0x6f, 0x88, 0x54, 0x52, 0x7a, 0x07, 0x69, 0x0e, 0x96, 0x27}, + {0x14, 0xee, 0xca, 0x33, 0x8b, 0x20, 0x86, 0x13, 0x48, 0x5e, 0xa0, 0x30, 0x8f, 0xd7, 0xa1, 0x5e}, + {0xa1, 0xf1, 0xeb, 0xbe, 0xd8, 0xdb, 0xc1, 0x53, 0xc0, 0xb8, 0x4a, 0xa6, 0x1f, 0xf0, 0x82, 0x39}, + {0x3b, 0x62, 0xa9, 0xba, 0x62, 0x58, 0xf5, 0x61, 0x0f, 0x83, 0xe2, 0x64, 0xf3, 0x14, 0x97, 0xb4}, + {0x26, 0x44, 0x99, 0x06, 0x0a, 0xd9, 0xba, 0xab, 0xc4, 0x7f, 0x8b, 0x02, 0xbb, 0x6d, 0x71, 0xed}, + {0x00, 0x11, 0x0d, 0xc3, 0x78, 0x14, 0x69, 0x56, 0xc9, 0x54, 0x47, 0xd3, 0xf3, 0xd0, 0xfb, 0xba}, + {0x01, 0x51, 0xc5, 0x68, 0x38, 0x6b, 0x66, 0x77, 0xa2, 0xb4, 0xdc, 0x6f, 0x81, 0xe5, 0xdc, 0x18}, + {0xd6, 0x26, 0xb2, 0x66, 0x90, 0x5e, 0xf3, 0x58, 0x82, 0x63, 0x4d, 0xf6, 0x85, 0x32, 0xc1, 0x25}, + {0x98, 0x69, 0xe2, 0x47, 0xe9, 0xc0, 0x8b, 0x10, 0xd0, 0x29, 0x93, 0x4f, 0xc4, 0xb9, 0x52, 0xf7}, + {0x31, 0xfc, 0xef, 0xac, 0x66, 0xd7, 0xde, 0x9c, 0x7e, 0xc7, 0x48, 0x5f, 0xe4, 0x49, 0x49, 0x02}, + {0x54, 0x93, 0xe9, 0x99, 0x33, 0xb0, 0xa8, 0x11, 0x7e, 0x08, 0xec, 0x0f, 0x97, 0xcf, 0xc3, 0xd9}, + {0x6e, 0xe2, 0xa4, 0xca, 0x67, 0xb0, 0x54, 0xbb, 0xfd, 0x33, 0x15, 0xbf, 0x85, 0x23, 0x05, 0x77}, + {0x47, 0x3d, 0x06, 0xe8, 0x73, 0x8d, 0xb8, 0x98, 0x54, 0xc0, 0x66, 0xc4, 0x7a, 0xe4, 0x77, 0x40}, + {0xa4, 0x26, 0xe5, 0xe4, 0x23, 0xbf, 0x48, 0x85, 0x29, 0x4d, 0xa4, 0x81, 0xfe, 0xae, 0xf7, 0x23}, + {0x78, 0x01, 0x77, 0x31, 0xcf, 0x65, 0xfa, 0xb0, 0x74, 0xd5, 0x20, 0x89, 0x52, 0x51, 0x2e, 0xb1}, + {0x9e, 0x25, 0xfc, 0x83, 0x3f, 0x22, 0x90, 0x73, 0x3e, 0x93, 0x44, 0xa5, 0xe8, 0x38, 0x39, 0xeb}, + {0x56, 0x8e, 0x49, 0x5a, 0xbe, 0x52, 0x5a, 0x21, 0x8a, 0x22, 0x14, 0xcd, 0x3e, 0x07, 0x1d, 0x12}, + {0x4a, 0x29, 0xb5, 0x45, 0x52, 0xd1, 0x6b, 0x9a, 0x46, 0x9c, 0x10, 0x52, 0x8e, 0xff, 0x0a, 0xae}, + {0xc9, 0xd1, 0x84, 0xdd, 0xd5, 0xa9, 0xf5, 0xe0, 0xcf, 0x8c, 0xe2, 0x9a, 0x9a, 0xbf, 0x69, 0x1c}, + {0x2d, 0xb4, 0x79, 0xae, 0x78, 0xbd, 0x50, 0xd8, 0x88, 0x2a, 0x8a, 0x17, 0x8a, 0x61, 0x32, 0xad}, + {0x8e, 0xce, 0x5f, 0x04, 0x2d, 0x5e, 0x44, 0x7b, 0x50, 0x51, 0xb9, 0xea, 0xcb, 0x8d, 0x8f, 0x6f}, + {0x9c, 0x0b, 0x53, 0xb4, 0xb3, 0xc3, 0x07, 0xe8, 0x7e, 0xae, 0xe0, 0x86, 0x78, 0x14, 0x1f, 0x66}, + {0xab, 0xf2, 0x48, 0xaf, 0x69, 0xa6, 0xea, 0xe4, 0xbf, 0xd3, 0xeb, 0x2f, 0x12, 0x9e, 0xeb, 0x94}, + {0x06, 0x64, 0xda, 0x16, 0x68, 0x57, 0x4b, 0x88, 0xb9, 0x35, 0xf3, 0x02, 0x73, 0x58, 0xae, 0xf4}, + {0xaa, 0x4b, 0x9d, 0xc4, 0xbf, 0x33, 0x7d, 0xe9, 0x0c, 0xd4, 0xfd, 0x3c, 0x46, 0x7c, 0x6a, 0xb7}, + {0xea, 0x5c, 0x7f, 0x47, 0x1f, 0xaf, 0x6b, 0xde, 0x2b, 0x1a, 0xd7, 0xd4, 0x68, 0x6d, 0x22, 0x87}, + {0x29, 0x39, 0xb0, 0x18, 0x32, 0x23, 0xfa, 0xfc, 0x17, 0x23, 0xde, 0x4f, 0x52, 0xc4, 0x3d, 0x35}, + {0x7c, 0x39, 0x56, 0xca, 0x5e, 0xea, 0xfc, 0x3e, 0x36, 0x3e, 0x9d, 0x55, 0x65, 0x46, 0xeb, 0x68}, + {0x77, 0xc6, 0x07, 0x71, 0x46, 0xf0, 0x1c, 0x32, 0xb6, 0xb6, 0x9d, 0x5f, 0x4e, 0xa9, 0xff, 0xcf}, + {0x37, 0xa6, 0x98, 0x6c, 0xb8, 0x84, 0x7e, 0xdf, 0x09, 0x25, 0xf0, 0xf1, 0x30, 0x9b, 0x54, 0xde}, + {0xa7, 0x05, 0xf0, 0xe6, 0x9d, 0xa9, 0xa8, 0xf9, 0x07, 0x24, 0x1a, 0x2e, 0x92, 0x3c, 0x8c, 0xc8}, + {0x3d, 0xc4, 0x7d, 0x1f, 0x29, 0xc4, 0x48, 0x46, 0x1e, 0x9e, 0x76, 0xed, 0x90, 0x4f, 0x67, 0x11}, + {0x0d, 0x62, 0xbf, 0x01, 0xe6, 0xfc, 0x0e, 0x1a, 0x0d, 0x3c, 0x47, 0x51, 0xc5, 0xd3, 0x69, 0x2b}, + {0x8c, 0x03, 0x46, 0x8b, 0xca, 0x7c, 0x66, 0x9e, 0xe4, 0xfd, 0x5e, 0x08, 0x4b, 0xbe, 0xe7, 0xb5}, + {0x52, 0x8a, 0x5b, 0xb9, 0x3b, 0xaf, 0x2c, 0x9c, 0x44, 0x73, 0xcc, 0xe5, 0xd0, 0xd2, 0x2b, 0xd9}, + {0xdf, 0x6a, 0x30, 0x1e, 0x95, 0xc9, 0x5d, 0xad, 0x97, 0xae, 0x0c, 0xc8, 0xc6, 0x91, 0x3b, 0xd8}, + {0x80, 0x11, 0x89, 0x90, 0x2c, 0x85, 0x7f, 0x39, 0xe7, 0x35, 0x91, 0x28, 0x5e, 0x70, 0xb6, 0xdb}, + {0xe6, 0x17, 0x34, 0x6a, 0xc9, 0xc2, 0x31, 0xbb, 0x36, 0x50, 0xae, 0x34, 0xcc, 0xca, 0x0c, 0x5b}, + {0x27, 0xd9, 0x34, 0x37, 0xef, 0xb7, 0x21, 0xaa, 0x40, 0x18, 0x21, 0xdc, 0xec, 0x5a, 0xdf, 0x89}, + {0x89, 0x23, 0x7d, 0x9d, 0xed, 0x9c, 0x5e, 0x78, 0xd8, 0xb1, 0xc9, 0xb1, 0x66, 0xcc, 0x73, 0x42}, + {0x4a, 0x6d, 0x80, 0x91, 0xbf, 0x5e, 0x7d, 0x65, 0x11, 0x89, 0xfa, 0x94, 0xa2, 0x50, 0xb1, 0x4c}, + {0x0e, 0x33, 0xf9, 0x60, 0x55, 0xe7, 0xae, 0x89, 0x3f, 0xfc, 0x0e, 0x3d, 0xcf, 0x49, 0x29, 0x02}, + {0xe6, 0x1c, 0x43, 0x2b, 0x72, 0x0b, 0x19, 0xd1, 0x8e, 0xc8, 0xd8, 0x4b, 0xdc, 0x63, 0x15, 0x1b}, + {0xf7, 0xe5, 0xae, 0xf5, 0x49, 0xf7, 0x82, 0xcf, 0x37, 0x90, 0x55, 0xa6, 0x08, 0x26, 0x9b, 0x16}, + {0x43, 0x8d, 0x03, 0x0f, 0xd0, 0xb7, 0xa5, 0x4f, 0xa8, 0x37, 0xf2, 0xad, 0x20, 0x1a, 0x64, 0x03}, + {0xa5, 0x90, 0xd3, 0xee, 0x4f, 0xbf, 0x04, 0xe3, 0x24, 0x7e, 0x0d, 0x27, 0xf2, 0x86, 0x42, 0x3f}, + {0x5f, 0xe2, 0xc1, 0xa1, 0x72, 0xfe, 0x93, 0xc4, 0xb1, 0x5c, 0xd3, 0x7c, 0xae, 0xf9, 0xf5, 0x38}, + {0x2c, 0x97, 0x32, 0x5c, 0xbd, 0x06, 0xb3, 0x6e, 0xb2, 0x13, 0x3d, 0xd0, 0x8b, 0x3a, 0x01, 0x7c}, + {0x92, 0xc8, 0x14, 0x22, 0x7a, 0x6b, 0xca, 0x94, 0x9f, 0xf0, 0x65, 0x9f, 0x00, 0x2a, 0xd3, 0x9e}, + {0xdc, 0xe8, 0x50, 0x11, 0x0b, 0xd8, 0x32, 0x8c, 0xfb, 0xd5, 0x08, 0x41, 0xd6, 0x91, 0x1d, 0x87}, + {0x67, 0xf1, 0x49, 0x84, 0xc7, 0xda, 0x79, 0x12, 0x48, 0xe3, 0x2b, 0xb5, 0x92, 0x25, 0x83, 0xda}, + {0x19, 0x38, 0xf2, 0xcf, 0x72, 0xd5, 0x4e, 0xe9, 0x7e, 0x94, 0x16, 0x6f, 0xa9, 0x1d, 0x2a, 0x36}, + {0x74, 0x48, 0x1e, 0x96, 0x46, 0xed, 0x49, 0xfe, 0x0f, 0x62, 0x24, 0x30, 0x16, 0x04, 0x69, 0x8e}, + {0x57, 0xfc, 0xa5, 0xde, 0x98, 0xa9, 0xd6, 0xd8, 0x00, 0x64, 0x38, 0xd0, 0x58, 0x3d, 0x8a, 0x1d}, + {0x9f, 0xec, 0xde, 0x1c, 0xef, 0xdc, 0x1c, 0xbe, 0xd4, 0x76, 0x36, 0x74, 0xd9, 0x57, 0x53, 0x59}, + {0xe3, 0x04, 0x0c, 0x00, 0xeb, 0x28, 0xf1, 0x53, 0x66, 0xca, 0x73, 0xcb, 0xd8, 0x72, 0xe7, 0x40}, + {0x76, 0x97, 0x00, 0x9a, 0x6a, 0x83, 0x1d, 0xfe, 0xcc, 0xa9, 0x1c, 0x59, 0x93, 0x67, 0x0f, 0x7a}, + {0x58, 0x53, 0x54, 0x23, 0x21, 0xf5, 0x67, 0xa0, 0x05, 0xd5, 0x47, 0xa4, 0xf0, 0x47, 0x59, 0xbd}, + {0x51, 0x50, 0xd1, 0x77, 0x2f, 0x50, 0x83, 0x4a, 0x50, 0x3e, 0x06, 0x9a, 0x97, 0x3f, 0xbd, 0x7c}, +} + +func TestSum64(t *testing.T) { + for i, v := range golden { + h := New(v.k) + h.Write(v.m) + if sum := h.Sum64(); sum != v.r { + t.Errorf(`%d: expected "%x", got "%x"`, i, v.r, sum) + } + } +} + +func TestSum(t *testing.T) { + var r [8]byte + for i, v := range golden { + binary.LittleEndian.PutUint64(r[:], v.r) + h := New(v.k) + h.Write(v.m) + if sum := h.Sum(nil); !bytes.Equal(sum, r[:]) { + t.Errorf(`%d: expected "%x", got "%x"`, i, r, sum) + } + } + + var k [16]byte + var in [64]byte + for i := range k { + k[i] = byte(i) + } + + for i := 0; i < 64; i++ { + in[i] = byte(i) + h := New(k[:]) + h.Write(in[:i]) + if sum := h.Sum(nil); !bytes.Equal(sum, goldenRef[i]) { + t.Errorf(`%d: expected "%x", got "%x"`, i, goldenRef[i], sum) + } + } +} + +func TestSumUnaligned(t *testing.T) { + const align = 8 + var k [16]byte + var in [64 + align]byte + for i := range k { + k[i] = byte(i) + } + + for a := 1; a < align; a++ { + for i := 0; i < 64; i++ { + in[a+i] = byte(i) + h := New(k[:]) + h.Write(in[a : a+i]) + if sum := h.Sum(nil); !bytes.Equal(sum, goldenRef[i]) { + t.Errorf(`%d: expected "%x", got "%x"`, i, goldenRef[i], sum) + } + } + } +} + +func TestSum128(t *testing.T) { + var k [16]byte + var in [64]byte + for i := range k { + k[i] = byte(i) + } + + for i := 0; i < 64; i++ { + in[i] = byte(i) + h := New128(k[:]) + h.Write(in[:i]) + if sum := h.Sum(nil); !bytes.Equal(sum, goldenRef128[i]) { + t.Errorf(`%d: expected "%x", got "%x"`, i, goldenRef128[i], sum) + } + } +} + +func TestHash(t *testing.T) { + var k0, k1 uint64 + for i, v := range golden { + k0 = binary.LittleEndian.Uint64(v.k[0:8]) + k1 = binary.LittleEndian.Uint64(v.k[8:16]) + if sum := Hash(k0, k1, v.m); sum != v.r { + t.Errorf(`%d: expected "%x", got "%x"`, i, v.r, sum) + } + } + + var k [16]byte + var in [64]byte + for i := range k { + k[i] = byte(i) + } + k0 = binary.LittleEndian.Uint64(k[0:8]) + k1 = binary.LittleEndian.Uint64(k[8:16]) + + for i := 0; i < 64; i++ { + in[i] = byte(i) + ref := binary.LittleEndian.Uint64(goldenRef[i]) + if sum := Hash(k0, k1, in[:i]); sum != ref { + t.Errorf(`%d: expected "%x", got "%x"`, i, ref, sum) + } + } +} + +func TestHashUnaligned(t *testing.T) { + const align = 8 + var k0, k1 uint64 + var k [16]byte + var in [64 + align]byte + + for i := range k { + k[i] = byte(i) + } + k0 = binary.LittleEndian.Uint64(k[0:8]) + k1 = binary.LittleEndian.Uint64(k[8:16]) + + for a := 1; a < align; a++ { + for i := 0; i < 64; i++ { + in[a+i] = byte(i) + ref := binary.LittleEndian.Uint64(goldenRef[i]) + if sum := Hash(k0, k1, in[a:a+i]); sum != ref { + t.Errorf(`%d: expected "%x", got "%x"`, i, ref, sum) + } + } + } +} + +func TestHash128(t *testing.T) { + var k0, k1 uint64 + + var k [16]byte + var in [64]byte + for i := range k { + k[i] = byte(i) + } + k0 = binary.LittleEndian.Uint64(k[0:8]) + k1 = binary.LittleEndian.Uint64(k[8:16]) + + for i := 0; i < 64; i++ { + in[i] = byte(i) + ref0 := binary.LittleEndian.Uint64(goldenRef128[i][0:]) + ref1 := binary.LittleEndian.Uint64(goldenRef128[i][8:]) + if sum0, sum1 := Hash128(k0, k1, in[:i]); sum0 != ref0 || sum1 != ref1 { + t.Errorf(`%d: expected "%x, %x", got "%x, %x"`, i, ref0, ref1, sum0, sum1) + } + } +} + +var ( + key = zeroKey + key0, key1 uint64 + bench = New(key) + bench128 = New128(key) + buf = make([]byte, 8<<10) +) + +func BenchmarkHash8(b *testing.B) { + b.SetBytes(8) + for i := 0; i < b.N; i++ { + Hash(key0, key1, buf[:8]) + } +} + +func BenchmarkHash16(b *testing.B) { + b.SetBytes(16) + for i := 0; i < b.N; i++ { + Hash(key0, key1, buf[:16]) + } +} + +func BenchmarkHash40(b *testing.B) { + b.SetBytes(40) + for i := 0; i < b.N; i++ { + Hash(key0, key1, buf[:40]) + } +} + +func BenchmarkHash64(b *testing.B) { + b.SetBytes(64) + for i := 0; i < b.N; i++ { + Hash(key0, key1, buf[:64]) + } +} + +func BenchmarkHash128(b *testing.B) { + b.SetBytes(128) + for i := 0; i < b.N; i++ { + Hash(key0, key1, buf[:128]) + } +} + +func BenchmarkHash1K(b *testing.B) { + b.SetBytes(1024) + for i := 0; i < b.N; i++ { + Hash(key0, key1, buf[:1024]) + } +} + +func BenchmarkHash1Kunaligned(b *testing.B) { + b.SetBytes(1024) + for i := 0; i < b.N; i++ { + Hash(key0, key1, buf[1:1025]) + } +} + +func BenchmarkHash8K(b *testing.B) { + b.SetBytes(int64(len(buf))) + for i := 0; i < b.N; i++ { + Hash(key0, key1, buf) + } +} + +func BenchmarkHash128_8(b *testing.B) { + b.SetBytes(8) + for i := 0; i < b.N; i++ { + Hash128(key0, key1, buf[:8]) + } +} + +func BenchmarkHash128_16(b *testing.B) { + b.SetBytes(16) + for i := 0; i < b.N; i++ { + Hash128(key0, key1, buf[:16]) + } +} + +func BenchmarkHash128_40(b *testing.B) { + b.SetBytes(40) + for i := 0; i < b.N; i++ { + Hash128(key0, key1, buf[:40]) + } +} + +func BenchmarkHash128_64(b *testing.B) { + b.SetBytes(64) + for i := 0; i < b.N; i++ { + Hash128(key0, key1, buf[:64]) + } +} + +func BenchmarkHash128_128(b *testing.B) { + b.SetBytes(128) + for i := 0; i < b.N; i++ { + Hash128(key0, key1, buf[:128]) + } +} + +func BenchmarkHash128_1K(b *testing.B) { + b.SetBytes(1024) + for i := 0; i < b.N; i++ { + Hash128(key0, key1, buf[:1024]) + } +} + +func BenchmarkHash128_8K(b *testing.B) { + b.SetBytes(int64(len(buf))) + for i := 0; i < b.N; i++ { + Hash128(key0, key1, buf) + } +} + +func BenchmarkFull8(b *testing.B) { + b.SetBytes(8) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[:8]) + bench.Sum64() + } +} + +func BenchmarkFull16(b *testing.B) { + b.SetBytes(16) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[:16]) + bench.Sum64() + } +} + +func BenchmarkFull40(b *testing.B) { + b.SetBytes(24) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[:16]) + bench.Sum64() + } +} + +func BenchmarkFull64(b *testing.B) { + b.SetBytes(64) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[:64]) + bench.Sum64() + } +} + +func BenchmarkFull128(b *testing.B) { + b.SetBytes(128) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[:64]) + bench.Sum64() + } +} + +func BenchmarkFull1K(b *testing.B) { + b.SetBytes(1024) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[:1024]) + bench.Sum64() + } +} + +func BenchmarkFull1Kunaligned(b *testing.B) { + b.SetBytes(1024) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf[1:1025]) + bench.Sum64() + } +} + +func BenchmarkFull8K(b *testing.B) { + b.SetBytes(int64(len(buf))) + for i := 0; i < b.N; i++ { + bench.Reset() + bench.Write(buf) + bench.Sum64() + } +} + +func BenchmarkFull128_8(b *testing.B) { + b.SetBytes(8) + for i := 0; i < b.N; i++ { + bench128.Reset() + bench128.Write(buf[:8]) + bench128.Sum(nil) + } +} + +func BenchmarkFull128_16(b *testing.B) { + b.SetBytes(16) + for i := 0; i < b.N; i++ { + bench128.Reset() + bench128.Write(buf[:16]) + bench128.Sum(nil) + } +} + +func BenchmarkFull128_40(b *testing.B) { + b.SetBytes(24) + for i := 0; i < b.N; i++ { + bench128.Reset() + bench128.Write(buf[:16]) + bench128.Sum(nil) + } +} + +func BenchmarkFull128_64(b *testing.B) { + b.SetBytes(64) + for i := 0; i < b.N; i++ { + bench128.Reset() + bench128.Write(buf[:64]) + bench128.Sum(nil) + } +} + +func BenchmarkFull128_128(b *testing.B) { + b.SetBytes(128) + for i := 0; i < b.N; i++ { + bench128.Reset() + bench128.Write(buf[:64]) + bench128.Sum(nil) + } +} + +func BenchmarkFull128_1K(b *testing.B) { + b.SetBytes(1024) + for i := 0; i < b.N; i++ { + bench128.Reset() + bench128.Write(buf[:1024]) + bench128.Sum(nil) + } +} + +func BenchmarkFull128_8K(b *testing.B) { + b.SetBytes(int64(len(buf))) + for i := 0; i < b.N; i++ { + bench128.Reset() + bench128.Write(buf) + bench128.Sum(nil) + } +} diff --git a/vendor/github.com/sasha-s/go-IBLT/LICENSE b/vendor/github.com/sasha-s/go-IBLT/LICENSE new file mode 100644 index 0000000000..8f71f43fee --- /dev/null +++ b/vendor/github.com/sasha-s/go-IBLT/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/sasha-s/go-IBLT/iblt.go b/vendor/github.com/sasha-s/go-IBLT/iblt.go new file mode 100644 index 0000000000..a6f7cd705a --- /dev/null +++ b/vendor/github.com/sasha-s/go-IBLT/iblt.go @@ -0,0 +1,413 @@ +package iblt + +// Invertible Bloom Lookup Table from +// What’s the Difference? +// Efficient Set Reconciliation without Prior Context +// David Eppstein1 Michael T. Goodrich1 Frank Uyeda2 George Varghese +// https://www.ics.uci.edu/~eppstein/pubs/EppGooUye-SIGCOMM-11.pdf +// IBFL with N cells (N>=50) and K=4 can safely recover diffs of size at least N/2. +// For large N the space overhead is less than 1.3 (so we can decode diffs of size of n * 0.77). + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "errors" + "log" + + "github.com/dchest/siphash" +) + +// Filter is totally NOT thread-safe! +type Filter struct { + mask uint64 + keySums []uint64 + valueSums []*bts + counts []int + seen bitset + buf []byte + shift uint16 + idx []int +} + +func (f *Filter) Clone() *Filter { + r := &Filter{ + mask: f.mask, + keySums: make([]uint64, f.N()), + valueSums: make([]*bts, 0, f.N()), + counts: make([]int, f.N()), + seen: make(bitset, len(f.seen)), + shift: f.shift, + idx: make([]int, len(f.idx)), + } + + copy(r.keySums, f.keySums) + for _, v := range f.keySums { + r.keySums = append(r.keySums, v) + } + for _, v := range f.valueSums { + b := make([]byte, len(v.b)) + copy(b, v.b) + r.valueSums = append(r.valueSums, &bts{b}) + } + copy(r.counts, f.counts) + return r +} + +type serializableFilter struct { + KeySums []uint64 + ValueSums [][]byte + Counts []int + K int +} + +func (f Filter) MarshalBinary() (data []byte, err error) { + s := serializableFilter{ + KeySums: f.keySums, + Counts: f.counts, + ValueSums: make([][]byte, len(f.valueSums)), + K: len(f.idx), + } + for k, v := range f.valueSums { + s.ValueSums[k] = v.b + } + buf := &bytes.Buffer{} + err = gob.NewEncoder(buf).Encode(s) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (f *Filter) UnmarshalBinary(data []byte) error { + var s serializableFilter + err := gob.NewDecoder(bytes.NewReader(data)).Decode(&s) + if err != nil { + return err + } + *f = *New(s.K, len(s.Counts)) + f.keySums = s.KeySums + f.counts = s.Counts + for k, v := range s.ValueSums { + f.valueSums[k].b = v + } + return nil +} + +// New constructs a new Filter. +func New(k, l int) *Filter { + if k >= 10 || k < 1 { + panic("k should be between 1 and 10") + } + if l/k < 2 { + panic("l should be at least 2*k") + } + if l&(l-1) != 0 { + panic("l should be a power of two") + } + var shift uint16 + for ll := l; ll != 0; ll >>= 1 { + shift++ + } + values := make([]*bts, l) + for i := range values { + values[i] = &bts{} + } + return &Filter{ + mask: uint64(l - 1), + keySums: make([]uint64, l), + valueSums: values, + counts: make([]int, l), + seen: newBitSet(l), + shift: shift, + idx: make([]int, k), + } +} + +func (f Filter) K() int { + return len(f.idx) +} + +func (f Filter) N() int { + return len(f.counts) +} + +func xor(a, b []byte) []byte { + if len(b) > len(a) { + return _xor(b, a) + } + return _xor(a, b) +} + +func (f *Filter) getIdxHash(b []byte) ([]int, uint64) { + hash := hash(b) + return f.getIdx(hash), hash +} + +func (f *Filter) getIdx(hash uint64) []int { + f.seen.ClearAll() + v := hash + bits := uint16(64) + for k := range f.idx { + for { + if bits < f.shift { + v = xorShiftStarRound(&hash) + bits = 64 + } + pos := int(v & f.mask) + v >>= f.shift + bits -= f.shift + if f.seen.Test(pos) { + continue + } + f.seen.Set(pos) + f.idx[k] = pos + break + } + } + return f.idx +} + +func (f *Filter) Add(b []byte) { + idx, hash := f.getIdxHash(b) + enc := f.encode(b) + for _, k := range idx { + f.counts[k] = f.counts[k] + 1 + f.keySums[k] = f.keySums[k] ^ hash + f.valueSums[k].xorInPlace(enc) + } +} + +func (f *Filter) Remove(b []byte) { + idx, hash := f.getIdxHash(b) + enc := f.encode(b) + for _, k := range idx { + f.counts[k] = f.counts[k] - 1 + f.keySums[k] = f.keySums[k] ^ hash + f.valueSums[k].xorInPlace(enc) + } + // TODO: consider cleaning up pure cells: + // the value could get long if a very long values were inserted and then removed. +} + +type Diff struct { + Added [][]byte + Removed [][]byte +} + +func (f Filter) Diff(other Filter, diff *Filter) error { + if f.shift != other.shift || f.shift != diff.shift { + return errors.New("sizes should match") + } + if len(f.idx) != len(other.idx) || len(f.idx) != len(diff.idx) { + return errors.New("ks should match") + } + for k := range f.counts { + diff.counts[k] = f.counts[k] - other.counts[k] + diff.keySums[k] = f.keySums[k] ^ other.keySums[k] + diff.valueSums[k] = &bts{xor(f.valueSums[k].b, other.valueSums[k].b)} + } + return nil +} + +// Inplace +func (f *Filter) Sub(other Filter) error { + if f.shift != other.shift { + return errors.New("sizes should match") + } + if len(f.idx) != len(other.idx) { + return errors.New("ks should match") + } + for k := range f.counts { + f.counts[k] = f.counts[k] - other.counts[k] + f.keySums[k] = f.keySums[k] ^ other.keySums[k] + f.valueSums[k].xorInPlace(other.valueSums[k].b) + } + return nil + +} + +// Decode is distructive! +// One can apply the diff produced (even in case of error) to restore the previous state. +func (f *Filter) Decode() (*Diff, error) { + pure := make([]int, len(f.counts)) + numZ := 0 + diff := &Diff{} + lp := 0 + for k := range f.counts { + c := f.counts[k] + switch c { + case 0: + if f.keySums[k] == 0 { + numZ++ + } + case -1, 1: + _, err := f.decode(f.valueSums[k].b) + if err != nil { + continue + } + pure[lp] = k + lp++ + } + } + head := 0 + tail := lp - 1 + for lp > 0 { + // Deque pop + lp-- + pos := pure[head] + c := f.counts[pos] + head++ + if head == len(f.counts) { + head = 0 + } + if c != 1 && c != -1 { + continue + } + dec, err := f.decode(f.valueSums[pos].b) + if err != nil { + continue + } + h := hash(dec) + if h == 0 || h != f.keySums[pos] { + continue + } + if c == 1 { + diff.Added = append(diff.Added, dec) + } else { + diff.Removed = append(diff.Removed, dec) + } + idx := f.getIdx(h) + val := f.valueSums[pos].b + numZ++ + for _, k := range idx { + if k != pos { + if f.keySums[k] == 0 && f.counts[k] == 0 { + continue + } + f.keySums[k] = f.keySums[k] ^ h + f.valueSums[k].xorInPlace(val) + f.counts[k] = f.counts[k] - c + c := f.counts[k] + switch c { + case 0: + if f.keySums[k] == 0 { + numZ++ + } + case -1, 1: + _, err := f.decode(f.valueSums[k].b) + if err != nil { + continue + } + + // Deque push + lp++ + tail++ + if tail == len(f.counts) { + tail = 0 + } + pure[tail] = k + } + } + } + f.keySums[pos] = 0 + f.counts[pos] = 0 + } + var err error + if numZ != len(f.counts) { + err = errors.New("failed to decode") + } + return diff, err +} + +func _xor(a, b []byte) []byte { + // len(a) >= len(b) + // TODO: use append to reduce allocations? + r := make([]byte, len(a)) + for i, v := range b { + r[i] = v ^ a[i] + } + copy(r[len(b):], a[len(b):]) + return r +} + +type bts struct { + b []byte +} + +func (b *bts) xorInPlace(a []byte) { + if len(b.b) < len(a) { + b.b = _xor(a, b.b) + return + } + for i, v := range a { + b.b[i] = b.b[i] ^ v + } +} + +// Self-delimited encoding. +func (f *Filter) encode(b []byte) []byte { + if len(b) >= 1<<16 { + log.Panicln("len(b) is too large", len(b)) + } + l := len(b) + 2 + if len(f.buf) < l { + f.buf = make([]byte, l) + } + binary.LittleEndian.PutUint16(f.buf, uint16(len(b))) + copy(f.buf[2:], b) + return f.buf[:l] +} + +func (f *Filter) decode(b []byte) ([]byte, error) { + if len(b) == 0 { + return nil, nil + } + if len(b) < 2 { + return nil, errors.New("bad length") + } + l16 := binary.LittleEndian.Uint16(b) + l := int(l16) + if l+2 > len(b) { + return nil, errors.New("too short") + } + return b[2 : l+2], nil +} + +type bitset []uint64 + +func (b bitset) Set(i int) { + b[i>>6] |= 1 << (uint32(i) & 63) +} + +func (b bitset) Clear(i int) { + b[i>>6] &= ^(1 << (uint32(i) & 63)) +} + +func (b bitset) Test(i int) bool { + return (b[i>>6] & (1 << (uint32(i) & 63))) != 0 +} + +func (b bitset) ClearAll() { + for i := range b { + b[i] = 0 + } +} + +func newBitSet(l int) bitset { + return make([]uint64, (l+63)>>6) +} + +func hash(v []byte) uint64 { + return siphash.Hash(2, 57, v) +} + +func xorShiftStarRound(x *uint64) uint64 { + if *x == 0 { + *x = 1 + } + *x ^= (*x >> 12) + *x ^= (*x << 25) + *x ^= (*x >> 27) + return *x * 2685821657736338717 +} diff --git a/vendor/github.com/sasha-s/go-IBLT/iblt_test.go b/vendor/github.com/sasha-s/go-IBLT/iblt_test.go new file mode 100644 index 0000000000..07f15ad245 --- /dev/null +++ b/vendor/github.com/sasha-s/go-IBLT/iblt_test.go @@ -0,0 +1,275 @@ +package iblt + +import ( + "bytes" + "encoding/gob" + "fmt" + "math/rand" + "sort" + "testing" + + "github.com/davecgh/go-spew/spew" +) + +func TestIBLTSub(t *testing.T) { + var tcs = []struct { + a, b, add, remove []string + }{ + { + a: []string{"z"}, + add: []string{"z"}, + }, + { + a: []string{"z"}, + b: []string{"z"}, + }, + { + a: []string{"alpha", "beta", "gamma", "delta", "ε", "zeta", "η"}, + b: []string{"α", "beta", "gamma", "δ", "epsilon", "zeta", "η"}, + add: []string{"alpha", "delta", "ε"}, + remove: []string{"epsilon", "α", "δ"}, + }, + } + for _, l := range []int{16, 32, 1024} { + for _, tc := range tcs { + f := New(3, l) + f2 := New(3, l) + for _, s := range tc.a { + f.Add([]byte(s)) + } + for _, s := range tc.b { + f2.Add([]byte(s)) + } + for i := 0; i < 10000; i++ { + if i < 20 || rand.Intn(100) == 0 { + buf := &bytes.Buffer{} + err := gob.NewEncoder(buf).Encode(f) + if err != nil { + t.Error(err) + continue + } + var i2 Filter + err = gob.NewDecoder(buf).Decode(&i2) + if err != nil { + t.Error(err) + continue + } + if spew.Sdump(f.keySums) != spew.Sdump(i2.keySums) { + t.Error("decoded Filter is different, oops. keysums") + spew.Dump(f.keySums, ">>>>", i2.keySums) + } + if spew.Sdump(f.counts) != spew.Sdump(i2.counts) { + t.Error("decoded Filter is different, oops. counts") + spew.Dump(f.counts, ">>>>", i2.counts) + } + if spew.Sprint(f.valueSums) != spew.Sprint(i2.valueSums) { + t.Error("decoded Filter is different, oops. valueSums") + spew.Println("", f.valueSums, "\n>>>>\n", i2.valueSums) + } + if f.K() != i2.K() { + t.Errorf("k: expected %d, got %d", f.K(), i2.K()) + } + if f.N() != i2.N() { + t.Errorf("k: expected %d, got %d", f.N(), i2.N()) + } + + } + v := []byte(fmt.Sprint(i)) + if rand.Intn(3) == 0 { + f.Add(v) + f2.Add(v) + } else { + f.Remove(v) + f2.Remove(v) + } + } + if err := f.Sub(*f2); err != nil { + t.Error(spew.Sdump(tc), err) + continue + } + r, err := f2.Decode() + if err == nil { + t.Error(spew.Sdump(tc), err, spew.Sdump(f2, r)) + continue + } + r, err = f.Decode() + if err != nil { + t.Error(spew.Sdump(tc), err, spew.Sdump(f)) + continue + } + + sort.Strings(tc.add) + sort.Strings(tc.remove) + added := []string{} + for _, s := range r.Added { + added = append(added, string(s)) + } + removed := []string{} + for _, s := range r.Removed { + removed = append(removed, string(s)) + } + sort.Strings(added) + sort.Strings(removed) + if fmt.Sprint(tc.add) != fmt.Sprint(added) { + t.Error(spew.Sdump(tc), "|got", added, "|expected", tc.add) + } + if fmt.Sprint(tc.remove) != fmt.Sprint(removed) { + t.Error(spew.Sdump(tc), "|", removed, "|", tc.remove) + } + } + } +} + +func TestIBLT(t *testing.T) { + var tcs = []struct { + add, remove []string + }{ + { + add: []string{"z"}, + }, + { + remove: []string{"z"}, + }, + { + add: []string{"alpha", "beta", "gamma", "delta"}, + remove: []string{"omega", "z", "p", "q"}, + }, + } + for _, k := range []int{3, 5, 7} { + for _, l := range []int{16, 32, 1024} { + if k != 3 && l < 1024 { + continue + } + for _, tc := range tcs { + f := New(k, l) + for _, s := range tc.add { + f.Add([]byte(s)) + } + for _, s := range tc.remove { + f.Remove([]byte(s)) + } + r, err := f.Decode() + if err != nil { + t.Error(spew.Sdump(tc), err) + } + sort.Strings(tc.add) + sort.Strings(tc.remove) + added := []string{} + for _, s := range r.Added { + added = append(added, string(s)) + } + removed := []string{} + for _, s := range r.Removed { + removed = append(removed, string(s)) + } + sort.Strings(added) + sort.Strings(removed) + if fmt.Sprint(tc.add) != fmt.Sprint(added) { + t.Error(spew.Sdump(tc), "|", added, "|", tc.add) + } + if fmt.Sprint(tc.remove) != fmt.Sprint(removed) { + t.Error(spew.Sdump(tc), "|", removed, "|", tc.remove) + } + } + } + } +} + +func TestBitset(t *testing.T) { + a := 1 + for l := 1; l < 15000; l += a { + a += rand.Intn(100) + fmt.Print(".") + b := newBitSet(l) + b2 := map[int]bool{} + check := func() { + for pos := 0; pos < l; pos++ { + if b2[pos] != b.Test(pos) { + t.Fatal(pos, b2[pos], b.Test(pos)) + } + } + } + for k := 0; k < 10000; k++ { + pos := rand.Intn(l) + if rand.Intn(2) == 0 { + b.Clear(pos) + delete(b2, pos) + } else { + b.Set(pos) + b2[pos] = true + } + if rand.Intn(50) == 0 { + check() + } + } + check() + b.ClearAll() + for pos := 0; pos < l; pos++ { + if b.Test(pos) { + t.Fatal(pos, b.Test(pos)) + } + } + } +} + +func TestXOR(t *testing.T) { + tcs := []struct { + a, b, expected []byte + }{ + {[]byte{0}, nil, []byte{0}}, + {nil, []byte{0}, []byte{0}}, + {[]byte{0xfa}, []byte{0xff}, []byte{5}}, + {[]byte{0xfa, 0xff}, []byte{0xff}, []byte{5, 0xff}}, + {[]byte{0xfa, 0xff}, []byte{0xff, 0xff, 1}, []byte{5, 0, 1}}, + } + for _, tc := range tcs { + actual := xor(tc.a, tc.b) + if fmt.Sprint(actual) != fmt.Sprint(tc.expected) { + t.Errorf("`%v` ^ `%v`: expected `%v`, got `%v`\n", tc.a, tc.b, tc.expected, actual) + } + } + for _, tc := range tcs { + clone := func(x []byte) []byte { + y := make([]byte, len(x)) + copy(y, x) + return y + } + a := bts{clone(tc.a)} + a.xorInPlace(tc.b) + if fmt.Sprint(a.b) != fmt.Sprint(tc.expected) { + t.Errorf("`%v` ^ `%v`: expected `%v`, got `%v`\n", tc.a, tc.b, tc.expected, tc.a) + } + } +} + +func random(n int) []byte { + b := make([]byte, n) + for i := range b { + b[i] = byte(rand.Intn(256)) + } + return b +} + +func TestEncodeDecode(t *testing.T) { + var f Filter + for i := 0; i < 10000; i++ { + l := rand.Intn(30) + if i == 0 { + l = 1<<16 - 1 + } + b := random(l) + encoded := f.encode(b) + slack := rand.Intn(10) + e2 := append(encoded, random(slack)...) + decoded, _ := f.decode(e2) + if fmt.Sprint(decoded) != fmt.Sprint(b) { + t.Errorf("a := `%v`, a.endode() == `%v`, with slack: `%v`. a.endode().decode() ==`%v`\n", b, encoded, e2, decoded) + } + } + defer func() { + if recover() == nil { + t.Error("expected Panic") + } + }() + f.encode(random(1 << 16)) +} diff --git a/vendor/github.com/spaolacci/murmur3/LICENSE b/vendor/github.com/spaolacci/murmur3/LICENSE new file mode 100644 index 0000000000..2a46fd7500 --- /dev/null +++ b/vendor/github.com/spaolacci/murmur3/LICENSE @@ -0,0 +1,24 @@ +Copyright 2013, Sébastien Paolacci. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the library nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/spaolacci/murmur3/README.md b/vendor/github.com/spaolacci/murmur3/README.md new file mode 100644 index 0000000000..e463678a05 --- /dev/null +++ b/vendor/github.com/spaolacci/murmur3/README.md @@ -0,0 +1,86 @@ +murmur3 +======= + +[![Build Status](https://travis-ci.org/spaolacci/murmur3.svg?branch=master)](https://travis-ci.org/spaolacci/murmur3) + +Native Go implementation of Austin Appleby's third MurmurHash revision (aka +MurmurHash3). + +Reference algorithm has been slightly hacked as to support the streaming mode +required by Go's standard [Hash interface](http://golang.org/pkg/hash/#Hash). + + +Benchmarks +---------- + +Go tip as of 2014-06-12 (i.e almost go1.3), core i7 @ 3.4 Ghz. All runs +include hasher instantiation and sequence finalization. + +
+
+Benchmark32_1        500000000     7.69 ns/op      130.00 MB/s
+Benchmark32_2        200000000     8.83 ns/op      226.42 MB/s
+Benchmark32_4        500000000     7.99 ns/op      500.39 MB/s
+Benchmark32_8        200000000     9.47 ns/op      844.69 MB/s
+Benchmark32_16       100000000     12.1 ns/op     1321.61 MB/s
+Benchmark32_32       100000000     18.3 ns/op     1743.93 MB/s
+Benchmark32_64        50000000     30.9 ns/op     2071.64 MB/s
+Benchmark32_128       50000000     57.6 ns/op     2222.96 MB/s
+Benchmark32_256       20000000      116 ns/op     2188.60 MB/s
+Benchmark32_512       10000000      226 ns/op     2260.59 MB/s
+Benchmark32_1024       5000000      452 ns/op     2263.73 MB/s
+Benchmark32_2048       2000000      891 ns/op     2296.02 MB/s
+Benchmark32_4096       1000000     1787 ns/op     2290.92 MB/s
+Benchmark32_8192        500000     3593 ns/op     2279.68 MB/s
+Benchmark128_1       100000000     26.1 ns/op       38.33 MB/s
+Benchmark128_2       100000000     29.0 ns/op       69.07 MB/s
+Benchmark128_4        50000000     29.8 ns/op      134.17 MB/s
+Benchmark128_8        50000000     31.6 ns/op      252.86 MB/s
+Benchmark128_16      100000000     26.5 ns/op      603.42 MB/s
+Benchmark128_32      100000000     28.6 ns/op     1117.15 MB/s
+Benchmark128_64       50000000     35.5 ns/op     1800.97 MB/s
+Benchmark128_128      50000000     50.9 ns/op     2515.50 MB/s
+Benchmark128_256      20000000     76.9 ns/op     3330.11 MB/s
+Benchmark128_512      20000000      135 ns/op     3769.09 MB/s
+Benchmark128_1024     10000000      250 ns/op     4094.38 MB/s
+Benchmark128_2048      5000000      477 ns/op     4290.75 MB/s
+Benchmark128_4096      2000000      940 ns/op     4353.29 MB/s
+Benchmark128_8192      1000000     1838 ns/op     4455.47 MB/s
+
+
+ + +
+
+benchmark              Go1.0 MB/s    Go1.1 MB/s  speedup    Go1.2 MB/s  speedup    Go1.3 MB/s  speedup
+Benchmark32_1               98.90        118.59    1.20x        114.79    0.97x        130.00    1.13x
+Benchmark32_2              168.04        213.31    1.27x        210.65    0.99x        226.42    1.07x
+Benchmark32_4              414.01        494.19    1.19x        490.29    0.99x        500.39    1.02x
+Benchmark32_8              662.19        836.09    1.26x        836.46    1.00x        844.69    1.01x
+Benchmark32_16             917.46       1304.62    1.42x       1297.63    0.99x       1321.61    1.02x
+Benchmark32_32            1141.93       1737.54    1.52x       1728.24    0.99x       1743.93    1.01x
+Benchmark32_64            1289.47       2039.51    1.58x       2038.20    1.00x       2071.64    1.02x
+Benchmark32_128           1299.23       2097.63    1.61x       2177.13    1.04x       2222.96    1.02x
+Benchmark32_256           1369.90       2202.34    1.61x       2213.15    1.00x       2188.60    0.99x
+Benchmark32_512           1399.56       2255.72    1.61x       2264.49    1.00x       2260.59    1.00x
+Benchmark32_1024          1410.90       2285.82    1.62x       2270.99    0.99x       2263.73    1.00x
+Benchmark32_2048          1422.14       2297.62    1.62x       2269.59    0.99x       2296.02    1.01x
+Benchmark32_4096          1420.53       2307.81    1.62x       2273.43    0.99x       2290.92    1.01x
+Benchmark32_8192          1424.79       2312.87    1.62x       2286.07    0.99x       2279.68    1.00x
+Benchmark128_1               8.32         30.15    3.62x         30.84    1.02x         38.33    1.24x
+Benchmark128_2              16.38         59.72    3.65x         59.37    0.99x         69.07    1.16x
+Benchmark128_4              32.26        112.96    3.50x        114.24    1.01x        134.17    1.17x
+Benchmark128_8              62.68        217.88    3.48x        218.18    1.00x        252.86    1.16x
+Benchmark128_16            128.47        451.57    3.51x        474.65    1.05x        603.42    1.27x
+Benchmark128_32            246.18        910.42    3.70x        871.06    0.96x       1117.15    1.28x
+Benchmark128_64            449.05       1477.64    3.29x       1449.24    0.98x       1800.97    1.24x
+Benchmark128_128           762.61       2222.42    2.91x       2217.30    1.00x       2515.50    1.13x
+Benchmark128_256          1179.92       3005.46    2.55x       2931.55    0.98x       3330.11    1.14x
+Benchmark128_512          1616.51       3590.75    2.22x       3592.08    1.00x       3769.09    1.05x
+Benchmark128_1024         1964.36       3979.67    2.03x       4034.01    1.01x       4094.38    1.01x
+Benchmark128_2048         2225.07       4156.93    1.87x       4244.17    1.02x       4290.75    1.01x
+Benchmark128_4096         2360.15       4299.09    1.82x       4392.35    1.02x       4353.29    0.99x
+Benchmark128_8192         2411.50       4356.84    1.81x       4480.68    1.03x       4455.47    0.99x
+
+
+ diff --git a/vendor/github.com/spaolacci/murmur3/murmur.go b/vendor/github.com/spaolacci/murmur3/murmur.go new file mode 100644 index 0000000000..1252cf73a7 --- /dev/null +++ b/vendor/github.com/spaolacci/murmur3/murmur.go @@ -0,0 +1,64 @@ +// Copyright 2013, Sébastien Paolacci. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package murmur3 implements Austin Appleby's non-cryptographic MurmurHash3. + + Reference implementation: + http://code.google.com/p/smhasher/wiki/MurmurHash3 + + History, characteristics and (legacy) perfs: + https://sites.google.com/site/murmurhash/ + https://sites.google.com/site/murmurhash/statistics +*/ +package murmur3 + +type bmixer interface { + bmix(p []byte) (tail []byte) + Size() (n int) + reset() +} + +type digest struct { + clen int // Digested input cumulative length. + tail []byte // 0 to Size()-1 bytes view of `buf'. + buf [16]byte // Expected (but not required) to be Size() large. + seed uint32 // Seed for initializing the hash. + bmixer +} + +func (d *digest) BlockSize() int { return 1 } + +func (d *digest) Write(p []byte) (n int, err error) { + n = len(p) + d.clen += n + + if len(d.tail) > 0 { + // Stick back pending bytes. + nfree := d.Size() - len(d.tail) // nfree ∈ [1, d.Size()-1]. + if nfree < len(p) { + // One full block can be formed. + block := append(d.tail, p[:nfree]...) + p = p[nfree:] + _ = d.bmix(block) // No tail. + } else { + // Tail's buf is large enough to prevent reallocs. + p = append(d.tail, p...) + } + } + + d.tail = d.bmix(p) + + // Keep own copy of the 0 to Size()-1 pending bytes. + nn := copy(d.buf[:], d.tail) + d.tail = d.buf[:nn] + + return n, nil +} + +func (d *digest) Reset() { + d.clen = 0 + d.tail = nil + d.bmixer.reset() +} diff --git a/vendor/github.com/spaolacci/murmur3/murmur128.go b/vendor/github.com/spaolacci/murmur3/murmur128.go new file mode 100644 index 0000000000..a4b618b5f3 --- /dev/null +++ b/vendor/github.com/spaolacci/murmur3/murmur128.go @@ -0,0 +1,203 @@ +package murmur3 + +import ( + //"encoding/binary" + "hash" + "unsafe" +) + +const ( + c1_128 = 0x87c37b91114253d5 + c2_128 = 0x4cf5ad432745937f +) + +// Make sure interfaces are correctly implemented. +var ( + _ hash.Hash = new(digest128) + _ Hash128 = new(digest128) + _ bmixer = new(digest128) +) + +// Hash128 represents a 128-bit hasher +// Hack: the standard api doesn't define any Hash128 interface. +type Hash128 interface { + hash.Hash + Sum128() (uint64, uint64) +} + +// digest128 represents a partial evaluation of a 128 bites hash. +type digest128 struct { + digest + h1 uint64 // Unfinalized running hash part 1. + h2 uint64 // Unfinalized running hash part 2. +} + +// New128 returns a 128-bit hasher +func New128() Hash128 { return New128WithSeed(0) } + +// New128WithSeed returns a 128-bit hasher set with explicit seed value +func New128WithSeed(seed uint32) Hash128 { + d := new(digest128) + d.seed = seed + d.bmixer = d + d.Reset() + return d +} + +func (d *digest128) Size() int { return 16 } + +func (d *digest128) reset() { d.h1, d.h2 = uint64(d.seed), uint64(d.seed) } + +func (d *digest128) Sum(b []byte) []byte { + h1, h2 := d.Sum128() + return append(b, + byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32), + byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1), + + byte(h2>>56), byte(h2>>48), byte(h2>>40), byte(h2>>32), + byte(h2>>24), byte(h2>>16), byte(h2>>8), byte(h2), + ) +} + +func (d *digest128) bmix(p []byte) (tail []byte) { + h1, h2 := d.h1, d.h2 + + nblocks := len(p) / 16 + for i := 0; i < nblocks; i++ { + t := (*[2]uint64)(unsafe.Pointer(&p[i*16])) + k1, k2 := t[0], t[1] + + k1 *= c1_128 + k1 = (k1 << 31) | (k1 >> 33) // rotl64(k1, 31) + k1 *= c2_128 + h1 ^= k1 + + h1 = (h1 << 27) | (h1 >> 37) // rotl64(h1, 27) + h1 += h2 + h1 = h1*5 + 0x52dce729 + + k2 *= c2_128 + k2 = (k2 << 33) | (k2 >> 31) // rotl64(k2, 33) + k2 *= c1_128 + h2 ^= k2 + + h2 = (h2 << 31) | (h2 >> 33) // rotl64(h2, 31) + h2 += h1 + h2 = h2*5 + 0x38495ab5 + } + d.h1, d.h2 = h1, h2 + return p[nblocks*d.Size():] +} + +func (d *digest128) Sum128() (h1, h2 uint64) { + + h1, h2 = d.h1, d.h2 + + var k1, k2 uint64 + switch len(d.tail) & 15 { + case 15: + k2 ^= uint64(d.tail[14]) << 48 + fallthrough + case 14: + k2 ^= uint64(d.tail[13]) << 40 + fallthrough + case 13: + k2 ^= uint64(d.tail[12]) << 32 + fallthrough + case 12: + k2 ^= uint64(d.tail[11]) << 24 + fallthrough + case 11: + k2 ^= uint64(d.tail[10]) << 16 + fallthrough + case 10: + k2 ^= uint64(d.tail[9]) << 8 + fallthrough + case 9: + k2 ^= uint64(d.tail[8]) << 0 + + k2 *= c2_128 + k2 = (k2 << 33) | (k2 >> 31) // rotl64(k2, 33) + k2 *= c1_128 + h2 ^= k2 + + fallthrough + + case 8: + k1 ^= uint64(d.tail[7]) << 56 + fallthrough + case 7: + k1 ^= uint64(d.tail[6]) << 48 + fallthrough + case 6: + k1 ^= uint64(d.tail[5]) << 40 + fallthrough + case 5: + k1 ^= uint64(d.tail[4]) << 32 + fallthrough + case 4: + k1 ^= uint64(d.tail[3]) << 24 + fallthrough + case 3: + k1 ^= uint64(d.tail[2]) << 16 + fallthrough + case 2: + k1 ^= uint64(d.tail[1]) << 8 + fallthrough + case 1: + k1 ^= uint64(d.tail[0]) << 0 + k1 *= c1_128 + k1 = (k1 << 31) | (k1 >> 33) // rotl64(k1, 31) + k1 *= c2_128 + h1 ^= k1 + } + + h1 ^= uint64(d.clen) + h2 ^= uint64(d.clen) + + h1 += h2 + h2 += h1 + + h1 = fmix64(h1) + h2 = fmix64(h2) + + h1 += h2 + h2 += h1 + + return h1, h2 +} + +func fmix64(k uint64) uint64 { + k ^= k >> 33 + k *= 0xff51afd7ed558ccd + k ^= k >> 33 + k *= 0xc4ceb9fe1a85ec53 + k ^= k >> 33 + return k +} + +/* +func rotl64(x uint64, r byte) uint64 { + return (x << r) | (x >> (64 - r)) +} +*/ + +// Sum128 returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New128() +// hasher.Write(data) +// return hasher.Sum128() +func Sum128(data []byte) (h1 uint64, h2 uint64) { return Sum128WithSeed(data, 0) } + +// Sum128WithSeed returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New128WithSeed(seed) +// hasher.Write(data) +// return hasher.Sum128() +func Sum128WithSeed(data []byte, seed uint32) (h1 uint64, h2 uint64) { + d := &digest128{h1: uint64(seed), h2: uint64(seed)} + d.seed = seed + d.tail = d.bmix(data) + d.clen = len(data) + return d.Sum128() +} diff --git a/vendor/github.com/spaolacci/murmur3/murmur32.go b/vendor/github.com/spaolacci/murmur3/murmur32.go new file mode 100644 index 0000000000..e32c99511f --- /dev/null +++ b/vendor/github.com/spaolacci/murmur3/murmur32.go @@ -0,0 +1,167 @@ +package murmur3 + +// http://code.google.com/p/guava-libraries/source/browse/guava/src/com/google/common/hash/Murmur3_32HashFunction.java + +import ( + "hash" + "unsafe" +) + +// Make sure interfaces are correctly implemented. +var ( + _ hash.Hash = new(digest32) + _ hash.Hash32 = new(digest32) + _ bmixer = new(digest32) +) + +const ( + c1_32 uint32 = 0xcc9e2d51 + c2_32 uint32 = 0x1b873593 +) + +// digest32 represents a partial evaluation of a 32 bites hash. +type digest32 struct { + digest + h1 uint32 // Unfinalized running hash. +} + +// New32 returns new 32-bit hasher +func New32() hash.Hash32 { return New32WithSeed(0) } + +// New32WithSeed returns new 32-bit hasher set with explicit seed value +func New32WithSeed(seed uint32) hash.Hash32 { + d := new(digest32) + d.seed = seed + d.bmixer = d + d.Reset() + return d +} + +func (d *digest32) Size() int { return 4 } + +func (d *digest32) reset() { d.h1 = d.seed } + +func (d *digest32) Sum(b []byte) []byte { + h := d.Sum32() + return append(b, byte(h>>24), byte(h>>16), byte(h>>8), byte(h)) +} + +// Digest as many blocks as possible. +func (d *digest32) bmix(p []byte) (tail []byte) { + h1 := d.h1 + + nblocks := len(p) / 4 + for i := 0; i < nblocks; i++ { + k1 := *(*uint32)(unsafe.Pointer(&p[i*4])) + + k1 *= c1_32 + k1 = (k1 << 15) | (k1 >> 17) // rotl32(k1, 15) + k1 *= c2_32 + + h1 ^= k1 + h1 = (h1 << 13) | (h1 >> 19) // rotl32(h1, 13) + h1 = h1*4 + h1 + 0xe6546b64 + } + d.h1 = h1 + return p[nblocks*d.Size():] +} + +func (d *digest32) Sum32() (h1 uint32) { + + h1 = d.h1 + + var k1 uint32 + switch len(d.tail) & 3 { + case 3: + k1 ^= uint32(d.tail[2]) << 16 + fallthrough + case 2: + k1 ^= uint32(d.tail[1]) << 8 + fallthrough + case 1: + k1 ^= uint32(d.tail[0]) + k1 *= c1_32 + k1 = (k1 << 15) | (k1 >> 17) // rotl32(k1, 15) + k1 *= c2_32 + h1 ^= k1 + } + + h1 ^= uint32(d.clen) + + h1 ^= h1 >> 16 + h1 *= 0x85ebca6b + h1 ^= h1 >> 13 + h1 *= 0xc2b2ae35 + h1 ^= h1 >> 16 + + return h1 +} + +/* +func rotl32(x uint32, r byte) uint32 { + return (x << r) | (x >> (32 - r)) +} +*/ + +// Sum32 returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New32() +// hasher.Write(data) +// return hasher.Sum32() +func Sum32(data []byte) uint32 { return Sum32WithSeed(data, 0) } + +// Sum32WithSeed returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New32WithSeed(seed) +// hasher.Write(data) +// return hasher.Sum32() +func Sum32WithSeed(data []byte, seed uint32) uint32 { + + h1 := seed + + nblocks := len(data) / 4 + var p uintptr + if len(data) > 0 { + p = uintptr(unsafe.Pointer(&data[0])) + } + p1 := p + uintptr(4*nblocks) + for ; p < p1; p += 4 { + k1 := *(*uint32)(unsafe.Pointer(p)) + + k1 *= c1_32 + k1 = (k1 << 15) | (k1 >> 17) // rotl32(k1, 15) + k1 *= c2_32 + + h1 ^= k1 + h1 = (h1 << 13) | (h1 >> 19) // rotl32(h1, 13) + h1 = h1*4 + h1 + 0xe6546b64 + } + + tail := data[nblocks*4:] + + var k1 uint32 + switch len(tail) & 3 { + case 3: + k1 ^= uint32(tail[2]) << 16 + fallthrough + case 2: + k1 ^= uint32(tail[1]) << 8 + fallthrough + case 1: + k1 ^= uint32(tail[0]) + k1 *= c1_32 + k1 = (k1 << 15) | (k1 >> 17) // rotl32(k1, 15) + k1 *= c2_32 + h1 ^= k1 + } + + h1 ^= uint32(len(data)) + + h1 ^= h1 >> 16 + h1 *= 0x85ebca6b + h1 ^= h1 >> 13 + h1 *= 0xc2b2ae35 + h1 ^= h1 >> 16 + + return h1 +} diff --git a/vendor/github.com/spaolacci/murmur3/murmur64.go b/vendor/github.com/spaolacci/murmur3/murmur64.go new file mode 100644 index 0000000000..65a410ae0b --- /dev/null +++ b/vendor/github.com/spaolacci/murmur3/murmur64.go @@ -0,0 +1,57 @@ +package murmur3 + +import ( + "hash" +) + +// Make sure interfaces are correctly implemented. +var ( + _ hash.Hash = new(digest64) + _ hash.Hash64 = new(digest64) + _ bmixer = new(digest64) +) + +// digest64 is half a digest128. +type digest64 digest128 + +// New64 returns a 64-bit hasher +func New64() hash.Hash64 { return New64WithSeed(0) } + +// New64WithSeed returns a 64-bit hasher set with explicit seed value +func New64WithSeed(seed uint32) hash.Hash64 { + d := (*digest64)(New128WithSeed(seed).(*digest128)) + return d +} + +func (d *digest64) Sum(b []byte) []byte { + h1 := d.Sum64() + return append(b, + byte(h1>>56), byte(h1>>48), byte(h1>>40), byte(h1>>32), + byte(h1>>24), byte(h1>>16), byte(h1>>8), byte(h1)) +} + +func (d *digest64) Sum64() uint64 { + h1, _ := (*digest128)(d).Sum128() + return h1 +} + +// Sum64 returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New64() +// hasher.Write(data) +// return hasher.Sum64() +func Sum64(data []byte) uint64 { return Sum64WithSeed(data, 0) } + +// Sum64WithSeed returns the MurmurHash3 sum of data. It is equivalent to the +// following sequence (without the extra burden and the extra allocation): +// hasher := New64WithSeed(seed) +// hasher.Write(data) +// return hasher.Sum64() +func Sum64WithSeed(data []byte, seed uint32) uint64 { + d := &digest128{h1: uint64(seed), h2: uint64(seed)} + d.seed = seed + d.tail = d.bmix(data) + d.clen = len(data) + h1, _ := d.Sum128() + return h1 +} diff --git a/vendor/github.com/spaolacci/murmur3/murmur_test.go b/vendor/github.com/spaolacci/murmur3/murmur_test.go new file mode 100644 index 0000000000..94e2ee1b63 --- /dev/null +++ b/vendor/github.com/spaolacci/murmur3/murmur_test.go @@ -0,0 +1,185 @@ +package murmur3 + +import ( + "fmt" + "strconv" + "testing" +) + +var data = []struct { + seed uint32 + h32 uint32 + h64_1 uint64 + h64_2 uint64 + s string +}{ + {0x00, 0x00000000, 0x0000000000000000, 0x0000000000000000, ""}, + {0x00, 0x248bfa47, 0xcbd8a7b341bd9b02, 0x5b1e906a48ae1d19, "hello"}, + {0x00, 0x149bbb7f, 0x342fac623a5ebc8e, 0x4cdcbc079642414d, "hello, world"}, + {0x00, 0xe31e8a70, 0xb89e5988b737affc, 0x664fc2950231b2cb, "19 Jan 2038 at 3:14:07 AM"}, + {0x00, 0xd5c48bfc, 0xcd99481f9ee902c9, 0x695da1a38987b6e7, "The quick brown fox jumps over the lazy dog."}, + + {0x01, 0x514e28b7, 0x4610abe56eff5cb5, 0x51622daa78f83583, ""}, + {0x01, 0xbb4abcad, 0xa78ddff5adae8d10, 0x128900ef20900135, "hello"}, + {0x01, 0x6f5cb2e9, 0x8b95f808840725c6, 0x1597ed5422bd493b, "hello, world"}, + {0x01, 0xf50e1f30, 0x2a929de9c8f97b2f, 0x56a41d99af43a2db, "19 Jan 2038 at 3:14:07 AM"}, + {0x01, 0x846f6a36, 0xfb3325171f9744da, 0xaaf8b92a5f722952, "The quick brown fox jumps over the lazy dog."}, + + {0x2a, 0x087fcd5c, 0xf02aa77dfa1b8523, 0xd1016610da11cbb9, ""}, + {0x2a, 0xe2dbd2e1, 0xc4b8b3c960af6f08, 0x2334b875b0efbc7a, "hello"}, + {0x2a, 0x7ec7c6c2, 0xb91864d797caa956, 0xd5d139a55afe6150, "hello, world"}, + {0x2a, 0x58f745f6, 0xfd8f19ebdc8c6b6a, 0xd30fdc310fa08ff9, "19 Jan 2038 at 3:14:07 AM"}, + {0x2a, 0xc02d1434, 0x74f33c659cda5af7, 0x4ec7a891caf316f0, "The quick brown fox jumps over the lazy dog."}, +} + +func TestRefStrings(t *testing.T) { + for _, elem := range data { + + h32 := New32WithSeed(elem.seed) + h32.Write([]byte(elem.s)) + if v := h32.Sum32(); v != elem.h32 { + t.Errorf("[Hash32] key: '%s', seed: '%d': 0x%x (want 0x%x)", elem.s, elem.seed, v, elem.h32) + } + + h32.Reset() + h32.Write([]byte(elem.s)) + target := fmt.Sprintf("%08x", elem.h32) + if p := fmt.Sprintf("%x", h32.Sum(nil)); p != target { + t.Errorf("[Hash32] key: '%s', seed: '%d': %s (want %s)", elem.s, elem.seed, p, target) + } + + if v := Sum32WithSeed([]byte(elem.s), elem.seed); v != elem.h32 { + t.Errorf("[Hash32] key '%s', seed: '%d': 0x%x (want 0x%x)", elem.s, elem.seed, v, elem.h32) + } + + h64 := New64WithSeed(elem.seed) + h64.Write([]byte(elem.s)) + if v := h64.Sum64(); v != elem.h64_1 { + t.Errorf("'[Hash64] key: '%s', seed: '%d': 0x%x (want 0x%x)", elem.s, elem.seed, v, elem.h64_1) + } + + h64.Reset() + h64.Write([]byte(elem.s)) + target = fmt.Sprintf("%016x", elem.h64_1) + if p := fmt.Sprintf("%x", h64.Sum(nil)); p != target { + t.Errorf("[Hash64] key: '%s', seed: '%d': %s (want %s)", elem.s, elem.seed, p, target) + } + + if v := Sum64WithSeed([]byte(elem.s), elem.seed); v != elem.h64_1 { + t.Errorf("[Hash64] key: '%s', seed: '%d': 0x%x (want 0x%x)", elem.s, elem.seed, v, elem.h64_1) + } + + h128 := New128WithSeed(elem.seed) + + h128.Write([]byte(elem.s)) + if v1, v2 := h128.Sum128(); v1 != elem.h64_1 || v2 != elem.h64_2 { + t.Errorf("[Hash128] key: '%s', seed: '%d': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, elem.seed, v1, v2, elem.h64_1, elem.h64_2) + } + + h128.Reset() + h128.Write([]byte(elem.s)) + target = fmt.Sprintf("%016x%016x", elem.h64_1, elem.h64_2) + if p := fmt.Sprintf("%x", h128.Sum(nil)); p != target { + t.Errorf("[Hash128] key: '%s', seed: '%d': %s (want %s)", elem.s, elem.seed, p, target) + } + + if v1, v2 := Sum128WithSeed([]byte(elem.s), elem.seed); v1 != elem.h64_1 || v2 != elem.h64_2 { + t.Errorf("[Hash128] key: '%s', seed: '%d': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, elem.seed, v1, v2, elem.h64_1, elem.h64_2) + } + } +} + +func TestIncremental(t *testing.T) { + for _, elem := range data { + h32 := New32WithSeed(elem.seed) + h128 := New128WithSeed(elem.seed) + var i, j int + for k := len(elem.s); i < k; i = j { + j = 2*i + 3 + if j > k { + j = k + } + s := elem.s[i:j] + print(s + "|") + h32.Write([]byte(s)) + h128.Write([]byte(s)) + } + println() + if v := h32.Sum32(); v != elem.h32 { + t.Errorf("[Hash32] key: '%s', seed: '%d': 0x%x (want 0x%x)", elem.s, elem.seed, v, elem.h32) + } + if v1, v2 := h128.Sum128(); v1 != elem.h64_1 || v2 != elem.h64_2 { + t.Errorf("[Hash128] key: '%s', seed: '%d': 0x%x-0x%x (want 0x%x-0x%x)", elem.s, elem.seed, v1, v2, elem.h64_1, elem.h64_2) + } + } +} + +func Benchmark32(b *testing.B) { + buf := make([]byte, 8192) + for length := 1; length <= cap(buf); length *= 2 { + b.Run(strconv.Itoa(length), func(b *testing.B) { + buf = buf[:length] + b.SetBytes(int64(length)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Sum32(buf) + } + }) + } +} + +func BenchmarkPartial32(b *testing.B) { + buf := make([]byte, 128) + for length := 8; length <= cap(buf); length *= 2 { + b.Run(strconv.Itoa(length), func(b *testing.B) { + buf = buf[:length] + b.SetBytes(int64(length)) + + start := (32 / 8) / 2 + chunks := 7 + k := length / chunks + tail := (length - start) % k + + b.ResetTimer() + for i := 0; i < b.N; i++ { + hasher := New32() + hasher.Write(buf[0:start]) + + for j := start; j+k <= length; j += k { + hasher.Write(buf[j : j+k]) + } + + hasher.Write(buf[length-tail:]) + hasher.Sum32() + } + }) + } +} + +func Benchmark64(b *testing.B) { + buf := make([]byte, 8192) + for length := 1; length <= cap(buf); length *= 2 { + b.Run(strconv.Itoa(length), func(b *testing.B) { + buf = buf[:length] + b.SetBytes(int64(length)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Sum64(buf) + } + }) + } +} + +func Benchmark128(b *testing.B) { + buf := make([]byte, 8192) + for length := 1; length <= cap(buf); length *= 2 { + b.Run(strconv.Itoa(length), func(b *testing.B) { + buf = buf[:length] + b.SetBytes(int64(length)) + b.ResetTimer() + for i := 0; i < b.N; i++ { + Sum128(buf) + } + }) + } +} diff --git a/vendor/github.com/willf/bitset/LICENSE b/vendor/github.com/willf/bitset/LICENSE new file mode 100644 index 0000000000..59cab8a939 --- /dev/null +++ b/vendor/github.com/willf/bitset/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2014 Will Fitzgerald. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/willf/bitset/Makefile b/vendor/github.com/willf/bitset/Makefile new file mode 100644 index 0000000000..10d5e0ac5a --- /dev/null +++ b/vendor/github.com/willf/bitset/Makefile @@ -0,0 +1,197 @@ +# MAKEFILE +# +# @author Nicola Asuni +# @link https://github.com/willf/bitset +# ------------------------------------------------------------------------------ + +# List special make targets that are not associated with files +.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke + +# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS). +SHELL=/bin/bash + +# CVS path (path to the parent dir containing the project) +CVSPATH=github.com/willf + +# Project owner +OWNER=willf + +# Project vendor +VENDOR=willf + +# Project name +PROJECT=bitset + +# Project version +VERSION=$(shell cat VERSION) + +# Name of RPM or DEB package +PKGNAME=${VENDOR}-${PROJECT} + +# Current directory +CURRENTDIR=$(shell pwd) + +# GO lang path +ifneq ($(GOPATH),) + ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),) + # the defined GOPATH is not valid + GOPATH= + endif +endif +ifeq ($(GOPATH),) + # extract the GOPATH + GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR))) +endif + +# --- MAKE TARGETS --- + +# Display general help about this command +help: + @echo "" + @echo "$(PROJECT) Makefile." + @echo "GOPATH=$(GOPATH)" + @echo "The following commands are available:" + @echo "" + @echo " make qa : Run all the tests" + @echo " make test : Run the unit tests" + @echo "" + @echo " make format : Format the source code" + @echo " make fmtcheck : Check if the source code has been formatted" + @echo " make vet : Check for suspicious constructs" + @echo " make lint : Check for style errors" + @echo " make coverage : Generate the coverage report" + @echo " make cyclo : Generate the cyclomatic complexity report" + @echo " make ineffassign : Detect ineffectual assignments" + @echo " make misspell : Detect commonly misspelled words in source files" + @echo " make structcheck : Find unused struct fields" + @echo " make varcheck : Find unused global variables and constants" + @echo " make errcheck : Check that error return values are used" + @echo " make gosimple : Suggest code simplifications" + @echo " make astscan : GO AST scanner" + @echo "" + @echo " make docs : Generate source code documentation" + @echo "" + @echo " make deps : Get the dependencies" + @echo " make clean : Remove any build artifact" + @echo " make nuke : Deletes any intermediate file" + @echo "" + +# Alias for help target +all: help + +# Run the unit tests +test: + @mkdir -p target/test + @mkdir -p target/report + GOPATH=$(GOPATH) \ + go test \ + -covermode=atomic \ + -bench=. \ + -race \ + -cpuprofile=target/report/cpu.out \ + -memprofile=target/report/mem.out \ + -mutexprofile=target/report/mutex.out \ + -coverprofile=target/report/coverage.out \ + -v ./... | \ + tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \ + test $${PIPESTATUS[0]} -eq 0 + +# Format the source code +format: + @find . -type f -name "*.go" -exec gofmt -s -w {} \; + +# Check if the source code has been formatted +fmtcheck: + @mkdir -p target + @find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff + @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; } + +# Check for syntax errors +vet: + GOPATH=$(GOPATH) go vet . + +# Check for style errors +lint: + GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint . + +# Generate the coverage report +coverage: + @mkdir -p target/report + GOPATH=$(GOPATH) \ + go tool cover -html=target/report/coverage.out -o target/report/coverage.html + +# Report cyclomatic complexity +cyclo: + @mkdir -p target/report + GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Detect ineffectual assignments +ineffassign: + @mkdir -p target/report + GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Detect commonly misspelled words in source files +misspell: + @mkdir -p target/report + GOPATH=$(GOPATH) misspell -error ./ | tee target/report/misspell.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Find unused struct fields +structcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt + +# Find unused global variables and constants +varcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt + +# Check that error return values are used +errcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt + +# Suggest code simplifications +gosimple: + @mkdir -p target/report + GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt + +# AST scanner +astscan: + @mkdir -p target/report + GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Generate source docs +docs: + @mkdir -p target/docs + nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 & + wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060` + @echo ''${PKGNAME}' Documentation ...' > target/docs/index.html + +# Alias to run all quality-assurance checks +qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan + +# --- INSTALL --- + +# Get the dependencies +deps: + GOPATH=$(GOPATH) go get ./... + GOPATH=$(GOPATH) go get github.com/golang/lint/golint + GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report + GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov + GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo + GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign + GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell + GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck + GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck + GOPATH=$(GOPATH) go get github.com/kisielk/errcheck + GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple + GOPATH=$(GOPATH) go get github.com/GoASTScanner/gas + +# Remove any build artifact +clean: + GOPATH=$(GOPATH) go clean ./... + +# Deletes any intermediate file +nuke: + rm -rf ./target + GOPATH=$(GOPATH) go clean -i ./... diff --git a/vendor/github.com/willf/bitset/README.md b/vendor/github.com/willf/bitset/README.md new file mode 100644 index 0000000000..6c62b20c6c --- /dev/null +++ b/vendor/github.com/willf/bitset/README.md @@ -0,0 +1,96 @@ +# bitset + +*Go language library to map between non-negative integers and boolean values* + +[![Master Build Status](https://secure.travis-ci.org/willf/bitset.png?branch=master)](https://travis-ci.org/willf/bitset?branch=master) +[![Master Coverage Status](https://coveralls.io/repos/willf/bitset/badge.svg?branch=master&service=github)](https://coveralls.io/github/willf/bitset?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) +[![GoDoc](https://godoc.org/github.com/willf/bitset?status.svg)](http://godoc.org/github.com/willf/bitset) + + +## Description + +Package bitset implements bitsets, a mapping between non-negative integers and boolean values. +It should be more efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing individual integers. + +But it also provides set intersection, union, difference, complement, and symmetric operations, as well as tests to check whether any, all, or no bits are set, and querying a bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the memory allocation is approximately Max bits, where Max is the largest set bit. BitSets are never shrunk. On creation, a hint can be given for the number of bits that will be used. + +Many of the methods, including Set, Clear, and Flip, return a BitSet pointer, which allows for chaining. + +### Example use: + +```go +package main + +import ( + "fmt" + "math/rand" + + "github.com/willf/bitset" +) + +func main() { + fmt.Printf("Hello from BitSet!\n") + var b bitset.BitSet + // play some Go Fish + for i := 0; i < 100; i++ { + card1 := uint(rand.Intn(52)) + card2 := uint(rand.Intn(52)) + b.Set(card1) + if b.Test(card2) { + fmt.Println("Go Fish!") + } + b.Clear(card1) + } + + // Chaining + b.Set(10).Set(11) + + for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { + fmt.Println("The following bit is set:", i) + } + if b.Intersection(bitset.New(100).Set(10)).Count() == 1 { + fmt.Println("Intersection works.") + } else { + fmt.Println("Intersection doesn't work???") + } +} +``` + +As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. + +Godoc documentation is at: https://godoc.org/github.com/willf/bitset + + +## Implementation Note + +Go 1.9 introduced a native `math/bits` library. We provide backward compatibility to Go 1.7, which might be removed. + +It is possible that a later version will match the `math/bits` return signature for counts (which is `int`, rather than our library's `unit64`). If so, the version will be bumped. + +## Installation + +```bash +go get github.com/willf/bitset +``` + +## Contributing + +If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") + +This project include a Makefile that allows you to test and build the project with simple commands. +To see all available options: +```bash +make help +``` + +## Running all tests + +Before committing the code, please check if it passes all tests using (note: this will install some dependencies): +```bash +make qa +``` diff --git a/vendor/github.com/willf/bitset/VERSION b/vendor/github.com/willf/bitset/VERSION new file mode 100644 index 0000000000..781dcb07cd --- /dev/null +++ b/vendor/github.com/willf/bitset/VERSION @@ -0,0 +1 @@ +1.1.3 diff --git a/vendor/github.com/willf/bitset/bitset.go b/vendor/github.com/willf/bitset/bitset.go new file mode 100644 index 0000000000..dbd17089aa --- /dev/null +++ b/vendor/github.com/willf/bitset/bitset.go @@ -0,0 +1,709 @@ +/* +Package bitset implements bitsets, a mapping +between non-negative integers and boolean values. It should be more +efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing +individual integers. + +But it also provides set intersection, union, difference, +complement, and symmetric operations, as well as tests to +check whether any, all, or no bits are set, and querying a +bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the +memory allocation is approximately Max bits, where Max is +the largest set bit. BitSets are never shrunk. On creation, +a hint can be given for the number of bits that will be used. + +Many of the methods, including Set,Clear, and Flip, return +a BitSet pointer, which allows for chaining. + +Example use: + + import "bitset" + var b BitSet + b.Set(10).Set(11) + if b.Test(1000) { + b.Clear(1000) + } + if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { + fmt.Println("Intersection works.") + } + +As an alternative to BitSets, one should check out the 'big' package, +which provides a (less set-theoretical) view of bitsets. + +*/ +package bitset + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" +) + +// the wordSize of a bit set +const wordSize = uint(64) + +// log2WordSize is lg(wordSize) +const log2WordSize = uint(6) + +// allBits has every bit set +const allBits uint64 = 0xffffffffffffffff + +// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. +type BitSet struct { + length uint + set []uint64 +} + +// Error is used to distinguish errors (panics) generated in this package. +type Error string + +// safeSet will fixup b.set to be non-nil and return the field value +func (b *BitSet) safeSet() []uint64 { + if b.set == nil { + b.set = make([]uint64, wordsNeeded(0)) + } + return b.set +} + +// From is a constructor used to create a BitSet from an array of integers +func From(buf []uint64) *BitSet { + return &BitSet{uint(len(buf)) * 64, buf} +} + +// Bytes returns the bitset as array of integers +func (b *BitSet) Bytes() []uint64 { + return b.set +} + +// wordsNeeded calculates the number of words needed for i bits +func wordsNeeded(i uint) int { + if i > (Cap() - wordSize + 1) { + return int(Cap() >> log2WordSize) + } + return int((i + (wordSize - 1)) >> log2WordSize) +} + +// New creates a new BitSet with a hint that length bits will be required +func New(length uint) (bset *BitSet) { + defer func() { + if r := recover(); r != nil { + bset = &BitSet{ + 0, + make([]uint64, 0), + } + } + }() + + bset = &BitSet{ + length, + make([]uint64, wordsNeeded(length)), + } + + return bset +} + +// Cap returns the total possible capacity, or number of bits +func Cap() uint { + return ^uint(0) +} + +// Len returns the length of the BitSet in words +func (b *BitSet) Len() uint { + return b.length +} + +// extendSetMaybe adds additional words to incorporate new bits if needed +func (b *BitSet) extendSetMaybe(i uint) { + if i >= b.length { // if we need more bits, make 'em + nsize := wordsNeeded(i + 1) + if b.set == nil { + b.set = make([]uint64, nsize) + } else if cap(b.set) >= nsize { + b.set = b.set[:nsize] // fast resize + } else if len(b.set) < nsize { + newset := make([]uint64, nsize, 2*nsize) // increase capacity 2x + copy(newset, b.set) + b.set = newset + } + b.length = i + 1 + } +} + +// Test whether bit i is set. +func (b *BitSet) Test(i uint) bool { + if i >= b.length { + return false + } + return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0 +} + +// Set bit i to 1 +func (b *BitSet) Set(i uint) *BitSet { + b.extendSetMaybe(i) + b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1)) + return b +} + +// Clear bit i to 0 +func (b *BitSet) Clear(i uint) *BitSet { + if i >= b.length { + return b + } + b.set[i>>log2WordSize] &^= 1 << (i & (wordSize - 1)) + return b +} + +// SetTo sets bit i to value +func (b *BitSet) SetTo(i uint, value bool) *BitSet { + if value { + return b.Set(i) + } + return b.Clear(i) +} + +// Flip bit at i +func (b *BitSet) Flip(i uint) *BitSet { + if i >= b.length { + return b.Set(i) + } + b.set[i>>log2WordSize] ^= 1 << (i & (wordSize - 1)) + return b +} + +// String creates a string representation of the Bitmap +func (b *BitSet) String() string { + // follows code from https://github.com/RoaringBitmap/roaring + var buffer bytes.Buffer + start := []byte("{") + buffer.Write(start) + counter := 0 + i, e := b.NextSet(0) + for e { + counter = counter + 1 + // to avoid exhausting the memory + if counter > 0x40000 { + buffer.WriteString("...") + break + } + buffer.WriteString(strconv.FormatInt(int64(i), 10)) + i, e = b.NextSet(i + 1) + if e { + buffer.WriteString(",") + } + } + buffer.WriteString("}") + return buffer.String() +} + +// NextSet returns the next bit set from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no set bit found) +// for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...} +func (b *BitSet) NextSet(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> (i & (wordSize - 1)) + if w != 0 { + return i + trailingZeroes64(w), true + } + x = x + 1 + for x < len(b.set) { + if b.set[x] != 0 { + return uint(x)*wordSize + trailingZeroes64(b.set[x]), true + } + x = x + 1 + + } + return 0, false +} + +// NextClear returns the next clear bit from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no bit found i.e. all bits are set) +func (b *BitSet) NextClear(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> (i & (wordSize - 1)) + wA := allBits >> (i & (wordSize - 1)) + index := i + trailingZeroes64(^w) + if w != wA && index < b.length { + return index, true + } + x++ + for x < len(b.set) { + index = uint(x)*wordSize + trailingZeroes64(^b.set[x]) + if b.set[x] != allBits && index < b.length { + return index, true + } + x++ + } + return 0, false +} + +// ClearAll clears the entire BitSet +func (b *BitSet) ClearAll() *BitSet { + if b != nil && b.set != nil { + for i := range b.set { + b.set[i] = 0 + } + } + return b +} + +// wordCount returns the number of words used in a bit set +func (b *BitSet) wordCount() int { + return len(b.set) +} + +// Clone this BitSet +func (b *BitSet) Clone() *BitSet { + c := New(b.length) + if b.set != nil { // Clone should not modify current object + copy(c.set, b.set) + } + return c +} + +// Copy into a destination BitSet +// Returning the size of the destination BitSet +// like array copy +func (b *BitSet) Copy(c *BitSet) (count uint) { + if c == nil { + return + } + if b.set != nil { // Copy should not modify current object + copy(c.set, b.set) + } + count = c.length + if b.length < c.length { + count = b.length + } + return +} + +// Count (number of set bits) +func (b *BitSet) Count() uint { + if b != nil && b.set != nil { + return uint(popcntSlice(b.set)) + } + return 0 +} + +// Equal tests the equvalence of two BitSets. +// False if they are of different sizes, otherwise true +// only if all the same bits are set +func (b *BitSet) Equal(c *BitSet) bool { + if c == nil { + return false + } + if b.length != c.length { + return false + } + if b.length == 0 { // if they have both length == 0, then could have nil set + return true + } + // testing for equality shoud not transform the bitset (no call to safeSet) + + for p, v := range b.set { + if c.set[p] != v { + return false + } + } + return true +} + +func panicIfNull(b *BitSet) { + if b == nil { + panic(Error("BitSet must not be null")) + } +} + +// Difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) Difference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + result = b.Clone() // clone b (in case b is bigger than compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + result.set[i] = b.set[i] &^ compare.set[i] + } + return +} + +// DifferenceCardinality computes the cardinality of the differnce +func (b *BitSet) DifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + cnt := uint64(0) + cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + cnt += popcntSlice(b.set[l:]) + return uint(cnt) +} + +// InPlaceDifference computes the difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet) InPlaceDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &^= compare.set[i] + } +} + +// Convenience function: return two bitsets ordered by +// increasing length. Note: neither can be nil +func sortByLength(a *BitSet, b *BitSet) (ap *BitSet, bp *BitSet) { + if a.length <= b.length { + ap, bp = a, b + } else { + ap, bp = b, a + } + return +} + +// Intersection of base set and other set +// This is the BitSet equivalent of & (and) +func (b *BitSet) Intersection(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = New(b.length) + for i, word := range b.set { + result.set[i] = word & compare.set[i] + } + return +} + +// IntersectionCardinality computes the cardinality of the union +func (b *BitSet) IntersectionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntAndSlice(b.set, compare.set) + return uint(cnt) +} + +// InPlaceIntersection destructively computes the intersection of +// base set and the compare set. +// This is the BitSet equivalent of & (and) +func (b *BitSet) InPlaceIntersection(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &= compare.set[i] + } + for i := l; i < len(b.set); i++ { + b.set[i] = 0 + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } +} + +// Union of base set and other set +// This is the BitSet equivalent of | (or) +func (b *BitSet) Union(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word | compare.set[i] + } + return +} + +// UnionCardinality computes the cardinality of the uniton of the base set +// and the compare set. +func (b *BitSet) UnionCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntOrSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceUnion creates the destructive union of base set and compare set. +// This is the BitSet equivalent of | (or). +func (b *BitSet) InPlaceUnion(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] |= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) SymmetricDifference(compare *BitSet) (result *BitSet) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + // compare is bigger, so clone it + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word ^ compare.set[i] + } + return +} + +// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference +func (b *BitSet) SymmetricDifferenceCardinality(compare *BitSet) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntXorSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 { + b.extendSetMaybe(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] ^= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// Is the length an exact multiple of word sizes? +func (b *BitSet) isLenExactMultiple() bool { + return b.length%wordSize == 0 +} + +// Clean last word by setting unused bits to 0 +func (b *BitSet) cleanLastWord() { + if !b.isLenExactMultiple() { + b.set[len(b.set)-1] &= allBits >> (wordSize - b.length%wordSize) + } +} + +// Complement computes the (local) complement of a biset (up to length bits) +func (b *BitSet) Complement() (result *BitSet) { + panicIfNull(b) + result = New(b.length) + for i, word := range b.set { + result.set[i] = ^word + } + result.cleanLastWord() + return +} + +// All returns true if all bits are set, false otherwise. Returns true for +// empty sets. +func (b *BitSet) All() bool { + panicIfNull(b) + return b.Count() == b.length +} + +// None returns true if no bit is set, false otherwise. Retursn true for +// empty sets. +func (b *BitSet) None() bool { + panicIfNull(b) + if b != nil && b.set != nil { + for _, word := range b.set { + if word > 0 { + return false + } + } + return true + } + return true +} + +// Any returns true if any bit is set, false otherwise +func (b *BitSet) Any() bool { + panicIfNull(b) + return !b.None() +} + +// IsSuperSet returns true if this is a superset of the other set +func (b *BitSet) IsSuperSet(other *BitSet) bool { + for i, e := other.NextSet(0); e; i, e = other.NextSet(i + 1) { + if !b.Test(i) { + return false + } + } + return true +} + +// IsStrictSuperSet returns true if this is a strict superset of the other set +func (b *BitSet) IsStrictSuperSet(other *BitSet) bool { + return b.Count() > other.Count() && b.IsSuperSet(other) +} + +// DumpAsBits dumps a bit set as a string of bits +func (b *BitSet) DumpAsBits() string { + if b.set == nil { + return "." + } + buffer := bytes.NewBufferString("") + i := len(b.set) - 1 + for ; i >= 0; i-- { + fmt.Fprintf(buffer, "%064b.", b.set[i]) + } + return string(buffer.Bytes()) +} + +// BinaryStorageSize returns the binary storage requirements +func (b *BitSet) BinaryStorageSize() int { + return binary.Size(uint64(0)) + binary.Size(b.set) +} + +// WriteTo writes a BitSet to a stream +func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { + length := uint64(b.length) + + // Write length + err := binary.Write(stream, binary.BigEndian, length) + if err != nil { + return 0, err + } + + // Write set + err = binary.Write(stream, binary.BigEndian, b.set) + return int64(b.BinaryStorageSize()), err +} + +// ReadFrom reads a BitSet from a stream written using WriteTo +func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { + var length uint64 + + // Read length first + err := binary.Read(stream, binary.BigEndian, &length) + if err != nil { + return 0, err + } + newset := New(uint(length)) + + if uint64(newset.length) != length { + return 0, errors.New("Unmarshalling error: type mismatch") + } + + // Read remaining bytes as set + err = binary.Read(stream, binary.BigEndian, newset.set) + if err != nil { + return 0, err + } + + *b = *newset + return int64(b.BinaryStorageSize()), nil +} + +// MarshalBinary encodes a BitSet into a binary form and returns the result. +func (b *BitSet) MarshalBinary() ([]byte, error) { + var buf bytes.Buffer + writer := bufio.NewWriter(&buf) + + _, err := b.WriteTo(writer) + if err != nil { + return []byte{}, err + } + + err = writer.Flush() + + return buf.Bytes(), err +} + +// UnmarshalBinary decodes the binary form generated by MarshalBinary. +func (b *BitSet) UnmarshalBinary(data []byte) error { + buf := bytes.NewReader(data) + reader := bufio.NewReader(buf) + + _, err := b.ReadFrom(reader) + + return err +} + +// MarshalJSON marshals a BitSet as a JSON structure +func (b *BitSet) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBuffer(make([]byte, 0, b.BinaryStorageSize())) + _, err := b.WriteTo(buffer) + if err != nil { + return nil, err + } + + // URLEncode all bytes + return json.Marshal(base64.URLEncoding.EncodeToString(buffer.Bytes())) +} + +// UnmarshalJSON unmarshals a BitSet from JSON created using MarshalJSON +func (b *BitSet) UnmarshalJSON(data []byte) error { + // Unmarshal as string + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + // URLDecode string + buf, err := base64.URLEncoding.DecodeString(s) + if err != nil { + return err + } + + _, err = b.ReadFrom(bytes.NewReader(buf)) + return err +} diff --git a/vendor/github.com/willf/bitset/bitset_benchmark_test.go b/vendor/github.com/willf/bitset/bitset_benchmark_test.go new file mode 100644 index 0000000000..f06d0a3c61 --- /dev/null +++ b/vendor/github.com/willf/bitset/bitset_benchmark_test.go @@ -0,0 +1,136 @@ +// Copyright 2014 Will Fitzgerald. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file tests bit sets + +package bitset + +import ( + "math/rand" + "testing" +) + +func BenchmarkSet(b *testing.B) { + b.StopTimer() + r := rand.New(rand.NewSource(0)) + sz := 100000 + s := New(uint(sz)) + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Set(uint(r.Int31n(int32(sz)))) + } +} + +func BenchmarkGetTest(b *testing.B) { + b.StopTimer() + r := rand.New(rand.NewSource(0)) + sz := 100000 + s := New(uint(sz)) + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Test(uint(r.Int31n(int32(sz)))) + } +} + +func BenchmarkSetExpand(b *testing.B) { + b.StopTimer() + sz := uint(100000) + b.StartTimer() + for i := 0; i < b.N; i++ { + var s BitSet + s.Set(sz) + } +} + +// go test -bench=Count +func BenchmarkCount(b *testing.B) { + b.StopTimer() + s := New(100000) + for i := 0; i < 100000; i += 100 { + s.Set(uint(i)) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Count() + } +} + +// go test -bench=Iterate +func BenchmarkIterate(b *testing.B) { + b.StopTimer() + s := New(10000) + for i := 0; i < 10000; i += 3 { + s.Set(uint(i)) + } + b.StartTimer() + for j := 0; j < b.N; j++ { + c := uint(0) + for i, e := s.NextSet(0); e; i, e = s.NextSet(i + 1) { + c++ + } + } +} + +// go test -bench=SparseIterate +func BenchmarkSparseIterate(b *testing.B) { + b.StopTimer() + s := New(100000) + for i := 0; i < 100000; i += 30 { + s.Set(uint(i)) + } + b.StartTimer() + for j := 0; j < b.N; j++ { + c := uint(0) + for i, e := s.NextSet(0); e; i, e = s.NextSet(i + 1) { + c++ + } + } +} + +// go test -bench=LemireCreate +// see http://lemire.me/blog/2016/09/22/swift-versus-java-the-bitset-performance-test/ +func BenchmarkLemireCreate(b *testing.B) { + for i := 0; i < b.N; i++ { + bitmap := New(0) // we force dynamic memory allocation + for v := uint(0); v <= 100000000; v += 100 { + bitmap.Set(v) + } + } +} + +// go test -bench=LemireCount +// see http://lemire.me/blog/2016/09/22/swift-versus-java-the-bitset-performance-test/ +func BenchmarkLemireCount(b *testing.B) { + bitmap := New(100000000) + for v := uint(0); v <= 100000000; v += 100 { + bitmap.Set(v) + } + b.ResetTimer() + sum := uint(0) + for i := 0; i < b.N; i++ { + sum += bitmap.Count() + } + if sum == 0 { // added just to fool ineffassign + return + } +} + +// go test -bench=LemireIterate +// see http://lemire.me/blog/2016/09/22/swift-versus-java-the-bitset-performance-test/ +func BenchmarkLemireIterate(b *testing.B) { + bitmap := New(100000000) + for v := uint(0); v <= 100000000; v += 100 { + bitmap.Set(v) + } + b.ResetTimer() + sum := uint(0) + for i := 0; i < b.N; i++ { + for i, e := bitmap.NextSet(0); e; i, e = bitmap.NextSet(i + 1) { + sum++ + } + } + if sum == 0 { // added just to fool ineffassign + return + } +} diff --git a/vendor/github.com/willf/bitset/bitset_test.go b/vendor/github.com/willf/bitset/bitset_test.go new file mode 100644 index 0000000000..bc419a0799 --- /dev/null +++ b/vendor/github.com/willf/bitset/bitset_test.go @@ -0,0 +1,1054 @@ +// Copyright 2014 Will Fitzgerald. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file tests bit sets + +package bitset + +import ( + "encoding" + "encoding/json" + "fmt" + "math" + "testing" +) + +func TestStringer(t *testing.T) { + v := New(0) + for i := uint(0); i < 10; i++ { + v.Set(i) + } + if v.String() != "{0,1,2,3,4,5,6,7,8,9}" { + t.Error("bad string output") + } + fmt.Println(v) +} + +func TestStringLong(t *testing.T) { + v := New(0) + for i := uint(0); i < 262145; i++ { + v.Set(i) + } + str := v.String() + if len(str) != 1723903 { + t.Error("Unexpected string length: ", len(str)) + } +} + +func TestEmptyBitSet(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Error("A zero-length bitset should be fine") + } + }() + b := New(0) + if b.Len() != 0 { + t.Errorf("Empty set should have capacity 0, not %d", b.Len()) + } +} + +func TestZeroValueBitSet(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Error("A zero-length bitset should be fine") + } + }() + var b BitSet + if b.Len() != 0 { + t.Errorf("Empty set should have capacity 0, not %d", b.Len()) + } +} + +func TestBitSetNew(t *testing.T) { + v := New(16) + if v.Test(0) { + t.Errorf("Unable to make a bit set and read its 0th value.") + } +} + +func TestBitSetHuge(t *testing.T) { + v := New(uint(math.MaxUint32)) + if v.Test(0) { + t.Errorf("Unable to make a huge bit set and read its 0th value.") + } +} + +func TestLen(t *testing.T) { + v := New(1000) + if v.Len() != 1000 { + t.Errorf("Len should be 1000, but is %d.", v.Len()) + } +} + +func TestBitSetIsClear(t *testing.T) { + v := New(1000) + for i := uint(0); i < 1000; i++ { + if v.Test(i) { + t.Errorf("Bit %d is set, and it shouldn't be.", i) + } + } +} + +func TestExendOnBoundary(t *testing.T) { + v := New(32) + defer func() { + if r := recover(); r != nil { + t.Error("Border out of index error should not have caused a panic") + } + }() + v.Set(32) +} + +func TestExpand(t *testing.T) { + v := New(0) + defer func() { + if r := recover(); r != nil { + t.Error("Expansion should not have caused a panic") + } + }() + for i := uint(0); i < 1000; i++ { + v.Set(i) + } +} + +func TestBitSetAndGet(t *testing.T) { + v := New(1000) + v.Set(100) + if !v.Test(100) { + t.Errorf("Bit %d is clear, and it shouldn't be.", 100) + } +} + +func TestNextClear(t *testing.T) { + v := New(1000) + v.Set(0).Set(1) + next, found := v.NextClear(0) + if !found || next != 2 { + t.Errorf("Found next clear bit as %d, it should have been 2", next) + } + + v = New(1000) + for i := uint(0); i < 66; i++ { + v.Set(i) + } + next, found = v.NextClear(0) + if !found || next != 66 { + t.Errorf("Found next clear bit as %d, it should have been 66", next) + } + + v = New(1000) + for i := uint(0); i < 64; i++ { + v.Set(i) + } + v.Clear(45) + v.Clear(52) + next, found = v.NextClear(10) + if !found || next != 45 { + t.Errorf("Found next clear bit as %d, it should have been 45", next) + } + + v = New(1000) + for i := uint(0); i < 128; i++ { + v.Set(i) + } + v.Clear(73) + v.Clear(99) + next, found = v.NextClear(10) + if !found || next != 73 { + t.Errorf("Found next clear bit as %d, it should have been 73", next) + } + + next, found = v.NextClear(72) + if !found || next != 73 { + t.Errorf("Found next clear bit as %d, it should have been 73", next) + } + next, found = v.NextClear(73) + if !found || next != 73 { + t.Errorf("Found next clear bit as %d, it should have been 73", next) + } + next, found = v.NextClear(74) + if !found || next != 99 { + t.Errorf("Found next clear bit as %d, it should have been 73", next) + } + + v = New(128) + next, found = v.NextClear(0) + if !found || next != 0 { + t.Errorf("Found next clear bit as %d, it should have been 0", next) + } + + for i := uint(0); i < 128; i++ { + v.Set(i) + } + _, found = v.NextClear(0) + if found { + t.Errorf("There are not clear bits") + } + + b := new(BitSet) + c, d := b.NextClear(1) + if c != 0 || d { + t.Error("Unexpected values") + return + } + + v = New(100) + for i := uint(0); i != 100; i++ { + v.Set(i) + } + next, found = v.NextClear(0) + if found || next != 0 { + t.Errorf("Found next clear bit as %d, it should have return (0, false)", next) + + } +} + +func TestIterate(t *testing.T) { + v := New(10000) + v.Set(0) + v.Set(1) + v.Set(2) + data := make([]uint, 3) + c := 0 + for i, e := v.NextSet(0); e; i, e = v.NextSet(i + 1) { + data[c] = i + c++ + } + if data[0] != 0 { + t.Errorf("bug 0") + } + if data[1] != 1 { + t.Errorf("bug 1") + } + if data[2] != 2 { + t.Errorf("bug 2") + } + v.Set(10) + v.Set(2000) + data = make([]uint, 5) + c = 0 + for i, e := v.NextSet(0); e; i, e = v.NextSet(i + 1) { + data[c] = i + c++ + } + if data[0] != 0 { + t.Errorf("bug 0") + } + if data[1] != 1 { + t.Errorf("bug 1") + } + if data[2] != 2 { + t.Errorf("bug 2") + } + if data[3] != 10 { + t.Errorf("bug 3") + } + if data[4] != 2000 { + t.Errorf("bug 4") + } + +} + +func TestSetTo(t *testing.T) { + v := New(1000) + v.SetTo(100, true) + if !v.Test(100) { + t.Errorf("Bit %d is clear, and it shouldn't be.", 100) + } + v.SetTo(100, false) + if v.Test(100) { + t.Errorf("Bit %d is set, and it shouldn't be.", 100) + } +} + +func TestChain(t *testing.T) { + if !New(1000).Set(100).Set(99).Clear(99).Test(100) { + t.Errorf("Bit %d is clear, and it shouldn't be.", 100) + } +} + +func TestOutOfBoundsLong(t *testing.T) { + v := New(64) + defer func() { + if r := recover(); r != nil { + t.Error("Long distance out of index error should not have caused a panic") + } + }() + v.Set(1000) +} + +func TestOutOfBoundsClose(t *testing.T) { + v := New(65) + defer func() { + if r := recover(); r != nil { + t.Error("Local out of index error should not have caused a panic") + } + }() + v.Set(66) +} + +func TestCount(t *testing.T) { + tot := uint(64*4 + 11) // just some multi unit64 number + v := New(tot) + checkLast := true + for i := uint(0); i < tot; i++ { + sz := uint(v.Count()) + if sz != i { + t.Errorf("Count reported as %d, but it should be %d", sz, i) + checkLast = false + break + } + v.Set(i) + } + if checkLast { + sz := uint(v.Count()) + if sz != tot { + t.Errorf("After all bits set, size reported as %d, but it should be %d", sz, tot) + } + } +} + +// test setting every 3rd bit, just in case something odd is happening +func TestCount2(t *testing.T) { + tot := uint(64*4 + 11) // just some multi unit64 number + v := New(tot) + for i := uint(0); i < tot; i += 3 { + sz := uint(v.Count()) + if sz != i/3 { + t.Errorf("Count reported as %d, but it should be %d", sz, i) + break + } + v.Set(i) + } +} + +// nil tests +func TestNullTest(t *testing.T) { + var v *BitSet + defer func() { + if r := recover(); r == nil { + t.Error("Checking bit of null reference should have caused a panic") + } + }() + v.Test(66) +} + +func TestNullSet(t *testing.T) { + var v *BitSet + defer func() { + if r := recover(); r == nil { + t.Error("Setting bit of null reference should have caused a panic") + } + }() + v.Set(66) +} + +func TestNullClear(t *testing.T) { + var v *BitSet + defer func() { + if r := recover(); r == nil { + t.Error("Clearning bit of null reference should have caused a panic") + } + }() + v.Clear(66) +} + +func TestNullCount(t *testing.T) { + var v *BitSet + defer func() { + if r := recover(); r != nil { + t.Error("Counting null reference should not have caused a panic") + } + }() + cnt := v.Count() + if cnt != 0 { + t.Errorf("Count reported as %d, but it should be 0", cnt) + } +} + +func TestPanicDifferenceBNil(t *testing.T) { + var b *BitSet + var compare = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil First should should have caused a panic") + } + }() + b.Difference(compare) +} + +func TestPanicDifferenceCompareNil(t *testing.T) { + var compare *BitSet + var b = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil Second should should have caused a panic") + } + }() + b.Difference(compare) +} + +func TestPanicUnionBNil(t *testing.T) { + var b *BitSet + var compare = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil First should should have caused a panic") + } + }() + b.Union(compare) +} + +func TestPanicUnionCompareNil(t *testing.T) { + var compare *BitSet + var b = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil Second should should have caused a panic") + } + }() + b.Union(compare) +} + +func TestPanicIntersectionBNil(t *testing.T) { + var b *BitSet + var compare = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil First should should have caused a panic") + } + }() + b.Intersection(compare) +} + +func TestPanicIntersectionCompareNil(t *testing.T) { + var compare *BitSet + var b = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil Second should should have caused a panic") + } + }() + b.Intersection(compare) +} + +func TestPanicSymmetricDifferenceBNil(t *testing.T) { + var b *BitSet + var compare = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil First should should have caused a panic") + } + }() + b.SymmetricDifference(compare) +} + +func TestPanicSymmetricDifferenceCompareNil(t *testing.T) { + var compare *BitSet + var b = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil Second should should have caused a panic") + } + }() + b.SymmetricDifference(compare) +} + +func TestPanicComplementBNil(t *testing.T) { + var b *BitSet + defer func() { + if r := recover(); r == nil { + t.Error("Nil should should have caused a panic") + } + }() + b.Complement() +} + +func TestPanicAnytBNil(t *testing.T) { + var b *BitSet + defer func() { + if r := recover(); r == nil { + t.Error("Nil should should have caused a panic") + } + }() + b.Any() +} + +func TestPanicNonetBNil(t *testing.T) { + var b *BitSet + defer func() { + if r := recover(); r == nil { + t.Error("Nil should should have caused a panic") + } + }() + b.None() +} + +func TestPanicAlltBNil(t *testing.T) { + var b *BitSet + defer func() { + if r := recover(); r == nil { + t.Error("Nil should should have caused a panic") + } + }() + b.All() +} + +func TestAll(t *testing.T) { + v := New(0) + if !v.All() { + t.Error("Empty sets should return true on All()") + } + v = New(2) + v.SetTo(0, true) + v.SetTo(1, true) + if !v.All() { + t.Error("Non-empty sets with all bits set should return true on All()") + } + v = New(2) + if v.All() { + t.Error("Non-empty sets with no bits set should return false on All()") + } + v = New(2) + v.SetTo(0, true) + if v.All() { + t.Error("Non-empty sets with some bits set should return false on All()") + } +} + +func TestNone(t *testing.T) { + v := New(0) + if !v.None() { + t.Error("Empty sets should return true on None()") + } + v = New(2) + v.SetTo(0, true) + v.SetTo(1, true) + if v.None() { + t.Error("Non-empty sets with all bits set should return false on None()") + } + v = New(2) + if !v.None() { + t.Error("Non-empty sets with no bits set should return true on None()") + } + v = New(2) + v.SetTo(0, true) + if v.None() { + t.Error("Non-empty sets with some bits set should return false on None()") + } + v = new(BitSet) + if !v.None() { + t.Error("Empty sets should return true on None()") + } +} + +func TestEqual(t *testing.T) { + a := New(100) + b := New(99) + c := New(100) + if a.Equal(b) { + t.Error("Sets of different sizes should be not be equal") + } + if !a.Equal(c) { + t.Error("Two empty sets of the same size should be equal") + } + a.Set(99) + c.Set(0) + if a.Equal(c) { + t.Error("Two sets with differences should not be equal") + } + c.Set(99) + a.Set(0) + if !a.Equal(c) { + t.Error("Two sets with the same bits set should be equal") + } + if a.Equal(nil) { + t.Error("The sets should be different") + } + a = New(0) + b = New(0) + if !a.Equal(b) { + t.Error("Two empty set should be equal") + } +} + +func TestUnion(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + if a.UnionCardinality(b) != 200 { + t.Errorf("Union should have 200 bits set, but had %d", a.UnionCardinality(b)) + } + if a.UnionCardinality(b) != b.UnionCardinality(a) { + t.Errorf("Union should be symmetric") + } + + c := a.Union(b) + d := b.Union(a) + if c.Count() != 200 { + t.Errorf("Union should have 200 bits set, but had %d", c.Count()) + } + if !c.Equal(d) { + t.Errorf("Union should be symmetric") + } +} + +func TestInPlaceUnion(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + c := a.Clone() + c.InPlaceUnion(b) + d := b.Clone() + d.InPlaceUnion(a) + if c.Count() != 200 { + t.Errorf("Union should have 200 bits set, but had %d", c.Count()) + } + if d.Count() != 200 { + t.Errorf("Union should have 200 bits set, but had %d", d.Count()) + } + if !c.Equal(d) { + t.Errorf("Union should be symmetric") + } +} + +func TestIntersection(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1).Set(i) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + if a.IntersectionCardinality(b) != 50 { + t.Errorf("Intersection should have 50 bits set, but had %d", a.IntersectionCardinality(b)) + } + if a.IntersectionCardinality(b) != b.IntersectionCardinality(a) { + t.Errorf("Intersection should be symmetric") + } + c := a.Intersection(b) + d := b.Intersection(a) + if c.Count() != 50 { + t.Errorf("Intersection should have 50 bits set, but had %d", c.Count()) + } + if !c.Equal(d) { + t.Errorf("Intersection should be symmetric") + } +} + +func TestInplaceIntersection(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1).Set(i) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + c := a.Clone() + c.InPlaceIntersection(b) + d := b.Clone() + d.InPlaceIntersection(a) + if c.Count() != 50 { + t.Errorf("Intersection should have 50 bits set, but had %d", c.Count()) + } + if d.Count() != 50 { + t.Errorf("Intersection should have 50 bits set, but had %d", d.Count()) + } + if !c.Equal(d) { + t.Errorf("Intersection should be symmetric") + } +} + +func TestDifference(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + if a.DifferenceCardinality(b) != 50 { + t.Errorf("a-b Difference should have 50 bits set, but had %d", a.DifferenceCardinality(b)) + } + if b.DifferenceCardinality(a) != 150 { + t.Errorf("b-a Difference should have 150 bits set, but had %d", b.DifferenceCardinality(a)) + } + + c := a.Difference(b) + d := b.Difference(a) + if c.Count() != 50 { + t.Errorf("a-b Difference should have 50 bits set, but had %d", c.Count()) + } + if d.Count() != 150 { + t.Errorf("b-a Difference should have 150 bits set, but had %d", d.Count()) + } + if c.Equal(d) { + t.Errorf("Difference, here, should not be symmetric") + } +} + +func TestInPlaceDifference(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + c := a.Clone() + c.InPlaceDifference(b) + d := b.Clone() + d.InPlaceDifference(a) + if c.Count() != 50 { + t.Errorf("a-b Difference should have 50 bits set, but had %d", c.Count()) + } + if d.Count() != 150 { + t.Errorf("b-a Difference should have 150 bits set, but had %d", d.Count()) + } + if c.Equal(d) { + t.Errorf("Difference, here, should not be symmetric") + } +} + +func TestSymmetricDifference(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) // 01010101010 ... 0000000 + b.Set(i - 1).Set(i) // 11111111111111111000000 + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + if a.SymmetricDifferenceCardinality(b) != 150 { + t.Errorf("a^b Difference should have 150 bits set, but had %d", a.SymmetricDifferenceCardinality(b)) + } + if b.SymmetricDifferenceCardinality(a) != 150 { + t.Errorf("b^a Difference should have 150 bits set, but had %d", b.SymmetricDifferenceCardinality(a)) + } + + c := a.SymmetricDifference(b) + d := b.SymmetricDifference(a) + if c.Count() != 150 { + t.Errorf("a^b Difference should have 150 bits set, but had %d", c.Count()) + } + if d.Count() != 150 { + t.Errorf("b^a Difference should have 150 bits set, but had %d", d.Count()) + } + if !c.Equal(d) { + t.Errorf("SymmetricDifference should be symmetric") + } +} + +func TestInPlaceSymmetricDifference(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) // 01010101010 ... 0000000 + b.Set(i - 1).Set(i) // 11111111111111111000000 + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + c := a.Clone() + c.InPlaceSymmetricDifference(b) + d := b.Clone() + d.InPlaceSymmetricDifference(a) + if c.Count() != 150 { + t.Errorf("a^b Difference should have 150 bits set, but had %d", c.Count()) + } + if d.Count() != 150 { + t.Errorf("b^a Difference should have 150 bits set, but had %d", d.Count()) + } + if !c.Equal(d) { + t.Errorf("SymmetricDifference should be symmetric") + } +} + +func TestComplement(t *testing.T) { + a := New(50) + b := a.Complement() + if b.Count() != 50 { + t.Errorf("Complement failed, size should be 50, but was %d", b.Count()) + } + a = New(50) + a.Set(10).Set(20).Set(42) + b = a.Complement() + if b.Count() != 47 { + t.Errorf("Complement failed, size should be 47, but was %d", b.Count()) + } +} + +func TestIsSuperSet(t *testing.T) { + a := New(500) + b := New(300) + c := New(200) + + // Setup bitsets + // a and b overlap + // only c is (strict) super set + for i := uint(0); i < 100; i++ { + a.Set(i) + } + for i := uint(50); i < 150; i++ { + b.Set(i) + } + for i := uint(0); i < 200; i++ { + c.Set(i) + } + + if a.IsSuperSet(b) { + t.Errorf("IsSuperSet fails") + } + if a.IsSuperSet(c) { + t.Errorf("IsSuperSet fails") + } + if b.IsSuperSet(a) { + t.Errorf("IsSuperSet fails") + } + if b.IsSuperSet(c) { + t.Errorf("IsSuperSet fails") + } + if !c.IsSuperSet(a) { + t.Errorf("IsSuperSet fails") + } + if !c.IsSuperSet(b) { + t.Errorf("IsSuperSet fails") + } + + if a.IsStrictSuperSet(b) { + t.Errorf("IsStrictSuperSet fails") + } + if a.IsStrictSuperSet(c) { + t.Errorf("IsStrictSuperSet fails") + } + if b.IsStrictSuperSet(a) { + t.Errorf("IsStrictSuperSet fails") + } + if b.IsStrictSuperSet(c) { + t.Errorf("IsStrictSuperSet fails") + } + if !c.IsStrictSuperSet(a) { + t.Errorf("IsStrictSuperSet fails") + } + if !c.IsStrictSuperSet(b) { + t.Errorf("IsStrictSuperSet fails") + } +} + +func TestDumpAsBits(t *testing.T) { + a := New(10).Set(10) + astr := "0000000000000000000000000000000000000000000000000000010000000000." + if a.DumpAsBits() != astr { + t.Errorf("DumpAsBits failed, output should be \"%s\" but was \"%s\"", astr, a.DumpAsBits()) + } + var b BitSet // zero value (b.set == nil) + bstr := "." + if b.DumpAsBits() != bstr { + t.Errorf("DumpAsBits failed, output should be \"%s\" but was \"%s\"", bstr, b.DumpAsBits()) + } +} + +func TestMarshalUnmarshalBinary(t *testing.T) { + a := New(1010).Set(10).Set(1001) + b := new(BitSet) + + copyBinary(t, a, b) + + // BitSets must be equal after marshalling and unmarshalling + if !a.Equal(b) { + t.Error("Bitsets are not equal:\n\t", a.DumpAsBits(), "\n\t", b.DumpAsBits()) + return + } +} + +func copyBinary(t *testing.T, from encoding.BinaryMarshaler, to encoding.BinaryUnmarshaler) { + data, err := from.MarshalBinary() + if err != nil { + t.Errorf(err.Error()) + return + } + + err = to.UnmarshalBinary(data) + if err != nil { + t.Errorf(err.Error()) + return + } +} + +func TestMarshalUnmarshalJSON(t *testing.T) { + a := New(1010).Set(10).Set(1001) + data, err := json.Marshal(a) + if err != nil { + t.Errorf(err.Error()) + return + } + + b := new(BitSet) + err = json.Unmarshal(data, b) + if err != nil { + t.Errorf(err.Error()) + return + } + + // Bitsets must be equal after marshalling and unmarshalling + if !a.Equal(b) { + t.Error("Bitsets are not equal:\n\t", a.DumpAsBits(), "\n\t", b.DumpAsBits()) + return + } +} + +func TestSafeSet(t *testing.T) { + b := new(BitSet) + c := b.safeSet() + outType := fmt.Sprintf("%T", c) + expType := "[]uint64" + if outType != expType { + t.Error("Expecting type: ", expType, ", gotf:", outType) + return + } + if len(c) != 0 { + t.Error("The slice should be empty") + return + } +} + +func TestFrom(t *testing.T) { + u := []uint64{2, 3, 5, 7, 11} + b := From(u) + outType := fmt.Sprintf("%T", b) + expType := "*bitset.BitSet" + if outType != expType { + t.Error("Expecting type: ", expType, ", gotf:", outType) + return + } +} + +func TestBytes(t *testing.T) { + b := new(BitSet) + c := b.Bytes() + outType := fmt.Sprintf("%T", c) + expType := "[]uint64" + if outType != expType { + t.Error("Expecting type: ", expType, ", gotf:", outType) + return + } + if len(c) != 0 { + t.Error("The slice should be empty") + return + } +} + +func TestCap(t *testing.T) { + c := Cap() + if c <= 0 { + t.Error("The uint capacity should be >= 0") + return + } +} + +func TestWordsNeededLong(t *testing.T) { + i := Cap() + out := wordsNeeded(i) + if out <= 0 { + t.Error("Unexpected value: ", out) + return + } +} + +func TestNewPanic(t *testing.T) { + n := New(Cap()) + if n.length != 0 { + t.Error("Unexpected value: ", n.length) + return + } +} + +func TestTestTooLong(t *testing.T) { + b := new(BitSet) + if b.Test(1) { + t.Error("Unexpected value: true") + return + } +} + +func TestClearTooLong(t *testing.T) { + b := new(BitSet) + c := b.Clear(1) + if b != c { + t.Error("Unexpected value") + return + } +} + +func TestClearAll(t *testing.T) { + u := []uint64{2, 3, 5, 7, 11} + b := From(u) + c := b.ClearAll() + if c.length != 320 { + t.Error("Unexpected length: ", b.length) + return + } + if c.Test(0) || c.Test(1) || c.Test(2) || c.Test(3) || c.Test(4) || c.Test(5) { + t.Error("All bits should be unset") + return + } +} + +func TestFlip(t *testing.T) { + b := new(BitSet) + c := b.Flip(11) + if c.length != 12 { + t.Error("Unexpected value: ", c.length) + return + } + d := c.Flip(7) + if d.length != 12 { + t.Error("Unexpected value: ", d.length) + return + } +} + +func TestCopy(t *testing.T) { + a := New(10) + if a.Copy(nil) != 0 { + t.Error("No values should be copied") + return + } + a = New(10) + b := New(20) + if a.Copy(b) != 10 { + t.Error("Unexpected value") + return + } +} + +func TestNextSetError(t *testing.T) { + b := new(BitSet) + c, d := b.NextSet(1) + if c != 0 || d { + t.Error("Unexpected values") + return + } +} diff --git a/vendor/github.com/willf/bitset/popcnt.go b/vendor/github.com/willf/bitset/popcnt.go new file mode 100644 index 0000000000..76577a8382 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt.go @@ -0,0 +1,53 @@ +package bitset + +// bit population count, take from +// https://code.google.com/p/go/issues/detail?id=4988#c11 +// credit: https://code.google.com/u/arnehormann/ +func popcount(x uint64) (n uint64) { + x -= (x >> 1) & 0x5555555555555555 + x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 + x += x >> 4 + x &= 0x0f0f0f0f0f0f0f0f + x *= 0x0101010101010101 + return x >> 56 +} + +func popcntSliceGo(s []uint64) uint64 { + cnt := uint64(0) + for _, x := range s { + cnt += popcount(x) + } + return cnt +} + +func popcntMaskSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] &^ m[i]) + } + return cnt +} + +func popcntAndSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] & m[i]) + } + return cnt +} + +func popcntOrSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] | m[i]) + } + return cnt +} + +func popcntXorSliceGo(s, m []uint64) uint64 { + cnt := uint64(0) + for i := range s { + cnt += popcount(s[i] ^ m[i]) + } + return cnt +} diff --git a/vendor/github.com/willf/bitset/popcnt_19.go b/vendor/github.com/willf/bitset/popcnt_19.go new file mode 100644 index 0000000000..fc8ff4f367 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_19.go @@ -0,0 +1,45 @@ +// +build go1.9 + +package bitset + +import "math/bits" + +func popcntSlice(s []uint64) uint64 { + var cnt int + for _, x := range s { + cnt += bits.OnesCount64(x) + } + return uint64(cnt) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount64(s[i] &^ m[i]) + } + return uint64(cnt) +} + +func popcntAndSlice(s, m []uint64) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount64(s[i] & m[i]) + } + return uint64(cnt) +} + +func popcntOrSlice(s, m []uint64) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount64(s[i] | m[i]) + } + return uint64(cnt) +} + +func popcntXorSlice(s, m []uint64) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount64(s[i] ^ m[i]) + } + return uint64(cnt) +} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.go b/vendor/github.com/willf/bitset/popcnt_amd64.go new file mode 100644 index 0000000000..4cf64f24ad --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64.go @@ -0,0 +1,68 @@ +// +build !go1.9 +// +build amd64,!appengine + +package bitset + +// *** the following functions are defined in popcnt_amd64.s + +//go:noescape + +func hasAsm() bool + +// useAsm is a flag used to select the GO or ASM implementation of the popcnt function +var useAsm = hasAsm() + +//go:noescape + +func popcntSliceAsm(s []uint64) uint64 + +//go:noescape + +func popcntMaskSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntAndSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntOrSliceAsm(s, m []uint64) uint64 + +//go:noescape + +func popcntXorSliceAsm(s, m []uint64) uint64 + +func popcntSlice(s []uint64) uint64 { + if useAsm { + return popcntSliceAsm(s) + } + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + if useAsm { + return popcntMaskSliceAsm(s, m) + } + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + if useAsm { + return popcntAndSliceAsm(s, m) + } + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + if useAsm { + return popcntOrSliceAsm(s, m) + } + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + if useAsm { + return popcntXorSliceAsm(s, m) + } + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/willf/bitset/popcnt_amd64.s b/vendor/github.com/willf/bitset/popcnt_amd64.s new file mode 100644 index 0000000000..666c0dcc17 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64.s @@ -0,0 +1,104 @@ +// +build !go1.9 +// +build amd64,!appengine + +TEXT ·hasAsm(SB),4,$0-1 +MOVQ $1, AX +CPUID +SHRQ $23, CX +ANDQ $1, CX +MOVB CX, ret+0(FP) +RET + +#define POPCNTQ_DX_DX BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0xd2 + +TEXT ·popcntSliceAsm(SB),4,$0-32 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntSliceEnd +popcntSliceLoop: +BYTE $0xf3; BYTE $0x48; BYTE $0x0f; BYTE $0xb8; BYTE $0x16 // POPCNTQ (SI), DX +ADDQ DX, AX +ADDQ $8, SI +LOOP popcntSliceLoop +popcntSliceEnd: +MOVQ AX, ret+24(FP) +RET + +TEXT ·popcntMaskSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntMaskSliceEnd +MOVQ m+24(FP), DI +popcntMaskSliceLoop: +MOVQ (DI), DX +NOTQ DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntMaskSliceLoop +popcntMaskSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntAndSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntAndSliceEnd +MOVQ m+24(FP), DI +popcntAndSliceLoop: +MOVQ (DI), DX +ANDQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntAndSliceLoop +popcntAndSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntOrSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntOrSliceEnd +MOVQ m+24(FP), DI +popcntOrSliceLoop: +MOVQ (DI), DX +ORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntOrSliceLoop +popcntOrSliceEnd: +MOVQ AX, ret+48(FP) +RET + +TEXT ·popcntXorSliceAsm(SB),4,$0-56 +XORQ AX, AX +MOVQ s+0(FP), SI +MOVQ s_len+8(FP), CX +TESTQ CX, CX +JZ popcntXorSliceEnd +MOVQ m+24(FP), DI +popcntXorSliceLoop: +MOVQ (DI), DX +XORQ (SI), DX +POPCNTQ_DX_DX +ADDQ DX, AX +ADDQ $8, SI +ADDQ $8, DI +LOOP popcntXorSliceLoop +popcntXorSliceEnd: +MOVQ AX, ret+48(FP) +RET diff --git a/vendor/github.com/willf/bitset/popcnt_amd64_test.go b/vendor/github.com/willf/bitset/popcnt_amd64_test.go new file mode 100644 index 0000000000..c79d00946b --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_amd64_test.go @@ -0,0 +1,79 @@ +// +build !go1.9 +// +build amd64,!appengine + +// This file tests the popcnt funtions + +package bitset + +import ( + "testing" +) + +func TestPopcntSliceCond(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + oldUseAsm := useAsm + defer func() { useAsm = oldUseAsm }() + useAsm = false + resGo := popcntSlice(s) + useAsm = (true && oldUseAsm) + resAsm := popcntSlice(s) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} + +func TestPopcntMaskSliceCond(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + oldUseAsm := useAsm + defer func() { useAsm = oldUseAsm }() + useAsm = false + resGo := popcntMaskSlice(s, m) + useAsm = (true && oldUseAsm) + resAsm := popcntMaskSlice(s, m) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} + +func TestPopcntAndSliceCond(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + oldUseAsm := useAsm + defer func() { useAsm = oldUseAsm }() + useAsm = false + resGo := popcntAndSlice(s, m) + useAsm = (true && oldUseAsm) + resAsm := popcntAndSlice(s, m) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} + +func TestPopcntOrSliceCond(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + oldUseAsm := useAsm + defer func() { useAsm = oldUseAsm }() + useAsm = false + resGo := popcntOrSlice(s, m) + useAsm = (true && oldUseAsm) + resAsm := popcntOrSlice(s, m) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} + +func TestPopcntXorSliceCond(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + oldUseAsm := useAsm + defer func() { useAsm = oldUseAsm }() + useAsm = false + resGo := popcntXorSlice(s, m) + useAsm = (true && oldUseAsm) + resAsm := popcntXorSlice(s, m) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} diff --git a/vendor/github.com/willf/bitset/popcnt_cmp_test.go b/vendor/github.com/willf/bitset/popcnt_cmp_test.go new file mode 100644 index 0000000000..8a06dc973b --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_cmp_test.go @@ -0,0 +1,59 @@ +// +build !go1.9 +// +build amd64,!appengine + +// This file tests the popcnt funtions + +package bitset + +import ( + "testing" +) + +func TestComparePopcntSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + resGo := popcntSliceGo(s) + resAsm := popcntSliceAsm(s) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} + +func TestComparePopcntMaskSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + resGo := popcntMaskSliceGo(s, m) + resAsm := popcntMaskSliceAsm(s, m) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} + +func TestComparePopcntAndSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + resGo := popcntAndSliceGo(s, m) + resAsm := popcntAndSliceAsm(s, m) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} + +func TestComparePopcntOrSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + resGo := popcntOrSliceGo(s, m) + resAsm := popcntOrSliceAsm(s, m) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} + +func TestComparePopcntXorSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + resGo := popcntXorSliceGo(s, m) + resAsm := popcntXorSliceAsm(s, m) + if resGo != resAsm { + t.Errorf("The implementations are different: GO %d != ASM %d", resGo, resAsm) + } +} diff --git a/vendor/github.com/willf/bitset/popcnt_generic.go b/vendor/github.com/willf/bitset/popcnt_generic.go new file mode 100644 index 0000000000..21e0ff7b4f --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_generic.go @@ -0,0 +1,24 @@ +// +build !go1.9 +// +build !amd64 appengine + +package bitset + +func popcntSlice(s []uint64) uint64 { + return popcntSliceGo(s) +} + +func popcntMaskSlice(s, m []uint64) uint64 { + return popcntMaskSliceGo(s, m) +} + +func popcntAndSlice(s, m []uint64) uint64 { + return popcntAndSliceGo(s, m) +} + +func popcntOrSlice(s, m []uint64) uint64 { + return popcntOrSliceGo(s, m) +} + +func popcntXorSlice(s, m []uint64) uint64 { + return popcntXorSliceGo(s, m) +} diff --git a/vendor/github.com/willf/bitset/popcnt_go18_test.go b/vendor/github.com/willf/bitset/popcnt_go18_test.go new file mode 100644 index 0000000000..df6921cd38 --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_go18_test.go @@ -0,0 +1,56 @@ +// This file tests the popcnt funtions + +package bitset + +import ( + "testing" +) + +func TestPopcntSliceGo(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + res := popcntSliceGo(s) + const l uint64 = 27 + if res != l { + t.Errorf("Wrong popcount %d != %d", res, l) + } +} + +func TestPopcntMaskSliceGo(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + res := popcntMaskSliceGo(s, m) + const l uint64 = 9 + if res != l { + t.Errorf("Wrong mask %d != %d", res, l) + } +} + +func TestPopcntAndSliceGo(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + res := popcntAndSliceGo(s, m) + const l uint64 = 18 + if res != l { + t.Errorf("Wrong And %d != %d", res, l) + } +} + +func TestPopcntOrSliceGo(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + res := popcntOrSliceGo(s, m) + const l uint64 = 50 + if res != l { + t.Errorf("Wrong OR %d != %d", res, l) + } +} + +func TestPopcntXorSliceGo(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + res := popcntXorSliceGo(s, m) + const l uint64 = 32 + if res != l { + t.Errorf("Wrong OR %d != %d", res, l) + } +} diff --git a/vendor/github.com/willf/bitset/popcnt_test.go b/vendor/github.com/willf/bitset/popcnt_test.go new file mode 100644 index 0000000000..d9fbb345fc --- /dev/null +++ b/vendor/github.com/willf/bitset/popcnt_test.go @@ -0,0 +1,56 @@ +// This file tests the popcnt funtions + +package bitset + +import ( + "testing" +) + +func TestPopcntSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + res := popcntSlice(s) + const l uint64 = 27 + if res != l { + t.Errorf("Wrong popcount %d != %d", res, l) + } +} + +func TestPopcntMaskSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + res := popcntMaskSlice(s, m) + const l uint64 = 9 + if res != l { + t.Errorf("Wrong mask %d != %d", res, l) + } +} + +func TestPopcntAndSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + res := popcntAndSlice(s, m) + const l uint64 = 18 + if res != l { + t.Errorf("Wrong And %d != %d", res, l) + } +} + +func TestPopcntOrSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + res := popcntOrSlice(s, m) + const l uint64 = 50 + if res != l { + t.Errorf("Wrong OR %d != %d", res, l) + } +} + +func TestPopcntXorSlice(t *testing.T) { + s := []uint64{2, 3, 5, 7, 11, 13, 17, 19, 23, 29} + m := []uint64{31, 37, 41, 43, 47, 53, 59, 61, 67, 71} + res := popcntXorSlice(s, m) + const l uint64 = 32 + if res != l { + t.Errorf("Wrong OR %d != %d", res, l) + } +} diff --git a/vendor/github.com/willf/bitset/trailing_zeros_18.go b/vendor/github.com/willf/bitset/trailing_zeros_18.go new file mode 100644 index 0000000000..c52b61be9f --- /dev/null +++ b/vendor/github.com/willf/bitset/trailing_zeros_18.go @@ -0,0 +1,14 @@ +// +build !go1.9 + +package bitset + +var deBruijn = [...]byte{ + 0, 1, 56, 2, 57, 49, 28, 3, 61, 58, 42, 50, 38, 29, 17, 4, + 62, 47, 59, 36, 45, 43, 51, 22, 53, 39, 33, 30, 24, 18, 12, 5, + 63, 55, 48, 27, 60, 41, 37, 16, 46, 35, 44, 21, 52, 32, 23, 11, + 54, 26, 40, 15, 34, 20, 31, 10, 25, 14, 19, 9, 13, 8, 7, 6, +} + +func trailingZeroes64(v uint64) uint { + return uint(deBruijn[((v&-v)*0x03f79d71b4ca8b09)>>58]) +} diff --git a/vendor/github.com/willf/bitset/trailing_zeros_19.go b/vendor/github.com/willf/bitset/trailing_zeros_19.go new file mode 100644 index 0000000000..36a988e714 --- /dev/null +++ b/vendor/github.com/willf/bitset/trailing_zeros_19.go @@ -0,0 +1,9 @@ +// +build go1.9 + +package bitset + +import "math/bits" + +func trailingZeroes64(v uint64) uint { + return uint(bits.TrailingZeros64(v)) +} diff --git a/vendor/github.com/willf/bloom/LICENSE b/vendor/github.com/willf/bloom/LICENSE new file mode 100644 index 0000000000..3b9d36aa6e --- /dev/null +++ b/vendor/github.com/willf/bloom/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2014 Will Fitzgerald. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/willf/bloom/Makefile b/vendor/github.com/willf/bloom/Makefile new file mode 100644 index 0000000000..88f2e34379 --- /dev/null +++ b/vendor/github.com/willf/bloom/Makefile @@ -0,0 +1,197 @@ +# MAKEFILE +# +# @author Nicola Asuni +# @link https://github.com/willf/bloom +# ------------------------------------------------------------------------------ + +# List special make targets that are not associated with files +.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke + +# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS). +SHELL=/bin/bash + +# CVS path (path to the parent dir containing the project) +CVSPATH=github.com/willf + +# Project owner +OWNER=willf + +# Project vendor +VENDOR=willf + +# Project name +PROJECT=bloom + +# Project version +VERSION=$(shell cat VERSION) + +# Name of RPM or DEB package +PKGNAME=${VENDOR}-${PROJECT} + +# Current directory +CURRENTDIR=$(shell pwd) + +# GO lang path +ifneq ($(GOPATH),) + ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),) + # the defined GOPATH is not valid + GOPATH= + endif +endif +ifeq ($(GOPATH),) + # extract the GOPATH + GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR))) +endif + +# --- MAKE TARGETS --- + +# Display general help about this command +help: + @echo "" + @echo "$(PROJECT) Makefile." + @echo "GOPATH=$(GOPATH)" + @echo "The following commands are available:" + @echo "" + @echo " make qa : Run all the tests" + @echo " make test : Run the unit tests" + @echo "" + @echo " make format : Format the source code" + @echo " make fmtcheck : Check if the source code has been formatted" + @echo " make vet : Check for suspicious constructs" + @echo " make lint : Check for style errors" + @echo " make coverage : Generate the coverage report" + @echo " make cyclo : Generate the cyclomatic complexity report" + @echo " make ineffassign : Detect ineffectual assignments" + @echo " make misspell : Detect commonly misspelled words in source files" + @echo " make structcheck : Find unused struct fields" + @echo " make varcheck : Find unused global variables and constants" + @echo " make errcheck : Check that error return values are used" + @echo " make gosimple : Suggest code simplifications" + @echo " make astscan : GO AST scanner" + @echo "" + @echo " make docs : Generate source code documentation" + @echo "" + @echo " make deps : Get the dependencies" + @echo " make clean : Remove any build artifact" + @echo " make nuke : Deletes any intermediate file" + @echo "" + +# Alias for help target +all: help + +# Run the unit tests +test: + @mkdir -p target/test + @mkdir -p target/report + GOPATH=$(GOPATH) \ + go test \ + -covermode=atomic \ + -bench=. \ + -race \ + -cpuprofile=target/report/cpu.out \ + -memprofile=target/report/mem.out \ + -mutexprofile=target/report/mutex.out \ + -coverprofile=target/report/coverage.out \ + -v ./... | \ + tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \ + test $${PIPESTATUS[0]} -eq 0 + +# Format the source code +format: + @find . -type f -name "*.go" -exec gofmt -s -w {} \; + +# Check if the source code has been formatted +fmtcheck: + @mkdir -p target + @find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff + @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; } + +# Check for syntax errors +vet: + GOPATH=$(GOPATH) go vet . + +# Check for style errors +lint: + GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint . + +# Generate the coverage report +coverage: + @mkdir -p target/report + GOPATH=$(GOPATH) \ + go tool cover -html=target/report/coverage.out -o target/report/coverage.html + +# Report cyclomatic complexity +cyclo: + @mkdir -p target/report + GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Detect ineffectual assignments +ineffassign: + @mkdir -p target/report + GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Detect commonly misspelled words in source files +misspell: + @mkdir -p target/report + GOPATH=$(GOPATH) misspell -error ./ | tee target/report/misspell.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Find unused struct fields +structcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt + +# Find unused global variables and constants +varcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt + +# Check that error return values are used +errcheck: + @mkdir -p target/report + GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt + +# Suggest code simplifications +gosimple: + @mkdir -p target/report + GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt + +# AST scanner +astscan: + @mkdir -p target/report + GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt ; test $${PIPESTATUS[0]} -eq 0 + +# Generate source docs +docs: + @mkdir -p target/docs + nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 & + wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060` + @echo ''${PKGNAME}' Documentation ...' > target/docs/index.html + +# Alias to run all quality-assurance checks +qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan + +# --- INSTALL --- + +# Get the dependencies +deps: + GOPATH=$(GOPATH) go get ./... + GOPATH=$(GOPATH) go get github.com/golang/lint/golint + GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report + GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov + GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo + GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign + GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell + GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck + GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck + GOPATH=$(GOPATH) go get github.com/kisielk/errcheck + GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple + GOPATH=$(GOPATH) go get github.com/GoASTScanner/gas + +# Remove any build artifact +clean: + GOPATH=$(GOPATH) go clean ./... + +# Deletes any intermediate file +nuke: + rm -rf ./target + GOPATH=$(GOPATH) go clean -i ./... diff --git a/vendor/github.com/willf/bloom/README.md b/vendor/github.com/willf/bloom/README.md new file mode 100644 index 0000000000..c554a35bf3 --- /dev/null +++ b/vendor/github.com/willf/bloom/README.md @@ -0,0 +1,69 @@ +Bloom filters +------------- + +[![Master Build Status](https://secure.travis-ci.org/willf/bloom.png?branch=master)](https://travis-ci.org/willf/bloom?branch=master) +[![Coverage Status](https://coveralls.io/repos/github/willf/bloom/badge.svg?branch=master)](https://coveralls.io/github/willf/bloom?branch=master) +[![Go Report Card](https://goreportcard.com/badge/github.com/willf/bloom)](https://goreportcard.com/report/github.com/willf/bloom) +[![GoDoc](https://godoc.org/github.com/willf/bloom?status.svg)](http://godoc.org/github.com/willf/bloom) + +A Bloom filter is a representation of a set of _n_ items, where the main +requirement is to make membership queries; _i.e._, whether an item is a +member of a set. + +A Bloom filter has two parameters: _m_, a maximum size (typically a reasonably large multiple of the cardinality of the set to represent) and _k_, the number of hashing functions on elements of the set. (The actual hashing functions are important, too, but this is not a parameter for this implementation). A Bloom filter is backed by a [BitSet](https://github.com/willf/bitset); a key is represented in the filter by setting the bits at each value of the hashing functions (modulo _m_). Set membership is done by _testing_ whether the bits at each value of the hashing functions (again, modulo _m_) are set. If so, the item is in the set. If the item is actually in the set, a Bloom filter will never fail (the true positive rate is 1.0); but it is susceptible to false positives. The art is to choose _k_ and _m_ correctly. + +In this implementation, the hashing functions used is [murmurhash](github.com/spaolacci/murmur3), a non-cryptographic hashing function. + +This implementation accepts keys for setting and testing as `[]byte`. Thus, to +add a string item, `"Love"`: + + n := uint(1000) + filter := bloom.New(20*n, 5) // load of 20, 5 keys + filter.Add([]byte("Love")) + +Similarly, to test if `"Love"` is in bloom: + + if filter.Test([]byte("Love")) + +For numeric data, I recommend that you look into the encoding/binary library. But, for example, to add a `uint32` to the filter: + + i := uint32(100) + n1 := make([]byte, 4) + binary.BigEndian.PutUint32(n1, i) + filter.Add(n1) + +Finally, there is a method to estimate the false positive rate of a particular +bloom filter for a set of size _n_: + + if filter.EstimateFalsePositiveRate(1000) > 0.001 + +Given the particular hashing scheme, it's best to be empirical about this. Note +that estimating the FP rate will clear the Bloom filter. + +Discussion here: [Bloom filter](https://groups.google.com/d/topic/golang-nuts/6MktecKi1bE/discussion) + +Godoc documentation: https://godoc.org/github.com/willf/bloom + +## Installation + +```bash +go get -u github.com/willf/bloom +``` + +## Contributing + +If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") + +This project include a Makefile that allows you to test and build the project with simple commands. +To see all available options: +```bash +make help +``` + +## Running all tests + +Before committing the code, please check if it passes all tests using (note: this will install some dependencies): +```bash +make deps +make qa +``` diff --git a/vendor/github.com/willf/bloom/VERSION b/vendor/github.com/willf/bloom/VERSION new file mode 100644 index 0000000000..50ffc5aa7f --- /dev/null +++ b/vendor/github.com/willf/bloom/VERSION @@ -0,0 +1 @@ +2.0.3 diff --git a/vendor/github.com/willf/bloom/bloom.go b/vendor/github.com/willf/bloom/bloom.go new file mode 100644 index 0000000000..743514f7ea --- /dev/null +++ b/vendor/github.com/willf/bloom/bloom.go @@ -0,0 +1,362 @@ +/* +Package bloom provides data structures and methods for creating Bloom filters. + +A Bloom filter is a representation of a set of _n_ items, where the main +requirement is to make membership queries; _i.e._, whether an item is a +member of a set. + +A Bloom filter has two parameters: _m_, a maximum size (typically a reasonably large +multiple of the cardinality of the set to represent) and _k_, the number of hashing +functions on elements of the set. (The actual hashing functions are important, too, +but this is not a parameter for this implementation). A Bloom filter is backed by +a BitSet; a key is represented in the filter by setting the bits at each value of the +hashing functions (modulo _m_). Set membership is done by _testing_ whether the +bits at each value of the hashing functions (again, modulo _m_) are set. If so, +the item is in the set. If the item is actually in the set, a Bloom filter will +never fail (the true positive rate is 1.0); but it is susceptible to false +positives. The art is to choose _k_ and _m_ correctly. + +In this implementation, the hashing functions used is murmurhash, +a non-cryptographic hashing function. + +This implementation accepts keys for setting as testing as []byte. Thus, to +add a string item, "Love": + + uint n = 1000 + filter := bloom.New(20*n, 5) // load of 20, 5 keys + filter.Add([]byte("Love")) + +Similarly, to test if "Love" is in bloom: + + if filter.Test([]byte("Love")) + +For numeric data, I recommend that you look into the binary/encoding library. But, +for example, to add a uint32 to the filter: + + i := uint32(100) + n1 := make([]byte,4) + binary.BigEndian.PutUint32(n1,i) + f.Add(n1) + +Finally, there is a method to estimate the false positive rate of a particular +Bloom filter for a set of size _n_: + + if filter.EstimateFalsePositiveRate(1000) > 0.001 + +Given the particular hashing scheme, it's best to be empirical about this. Note +that estimating the FP rate will clear the Bloom filter. +*/ +package bloom + +import ( + "bytes" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "math" + + "github.com/spaolacci/murmur3" + "github.com/willf/bitset" +) + +// A BloomFilter is a representation of a set of _n_ items, where the main +// requirement is to make membership queries; _i.e._, whether an item is a +// member of a set. +type BloomFilter struct { + m uint + k uint + b *bitset.BitSet +} + +func max(x, y uint) uint { + if x > y { + return x + } + return y +} + +// New creates a new Bloom filter with _m_ bits and _k_ hashing functions +// We force _m_ and _k_ to be at least one to avoid panics. +func New(m uint, k uint) *BloomFilter { + return &BloomFilter{max(1, m), max(1, k), bitset.New(m)} +} + +// From creates a new Bloom filter with len(_data_) * 64 bits and _k_ hashing +// functions. The data slice is not going to be reset. +func From(data []uint64, k uint) *BloomFilter { + m := uint(len(data) * 64) + return &BloomFilter{m, k, bitset.From(data)} +} + +// baseHashes returns the four hash values of data that are used to create k +// hashes +func baseHashes(data []byte) [4]uint64 { + a1 := []byte{1} // to grab another bit of data + hasher := murmur3.New128() + hasher.Write(data) // #nosec + v1, v2 := hasher.Sum128() + hasher.Write(a1) // #nosec + v3, v4 := hasher.Sum128() + return [4]uint64{ + v1, v2, v3, v4, + } +} + +// location returns the ith hashed location using the four base hash values +func location(h [4]uint64, i uint) uint64 { + ii := uint64(i) + return h[ii%2] + ii*h[2+(((ii+(ii%2))%4)/2)] +} + +// location returns the ith hashed location using the four base hash values +func (f *BloomFilter) location(h [4]uint64, i uint) uint { + return uint(location(h, i) % uint64(f.m)) +} + +// EstimateParameters estimates requirements for m and k. +// Based on https://bitbucket.org/ww/bloom/src/829aa19d01d9/bloom.go +// used with permission. +func EstimateParameters(n uint, p float64) (m uint, k uint) { + m = uint(math.Ceil(-1 * float64(n) * math.Log(p) / math.Pow(math.Log(2), 2))) + k = uint(math.Ceil(math.Log(2) * float64(m) / float64(n))) + return +} + +// NewWithEstimates creates a new Bloom filter for about n items with fp +// false positive rate +func NewWithEstimates(n uint, fp float64) *BloomFilter { + m, k := EstimateParameters(n, fp) + return New(m, k) +} + +// Cap returns the capacity, _m_, of a Bloom filter +func (f *BloomFilter) Cap() uint { + return f.m +} + +// K returns the number of hash functions used in the BloomFilter +func (f *BloomFilter) K() uint { + return f.k +} + +// Add data to the Bloom Filter. Returns the filter (allows chaining) +func (f *BloomFilter) Add(data []byte) *BloomFilter { + h := baseHashes(data) + for i := uint(0); i < f.k; i++ { + f.b.Set(f.location(h, i)) + } + return f +} + +// Merge the data from two Bloom Filters. +func (f *BloomFilter) Merge(g *BloomFilter) error { + // Make sure the m's and k's are the same, otherwise merging has no real use. + if f.m != g.m { + return fmt.Errorf("m's don't match: %d != %d", f.m, g.m) + } + + if f.k != g.k { + return fmt.Errorf("k's don't match: %d != %d", f.m, g.m) + } + + f.b.InPlaceUnion(g.b) + return nil +} + +// Copy creates a copy of a Bloom filter. +func (f *BloomFilter) Copy() *BloomFilter { + fc := New(f.m, f.k) + fc.Merge(f) // #nosec + return fc +} + +// AddString to the Bloom Filter. Returns the filter (allows chaining) +func (f *BloomFilter) AddString(data string) *BloomFilter { + return f.Add([]byte(data)) +} + +// Test returns true if the data is in the BloomFilter, false otherwise. +// If true, the result might be a false positive. If false, the data +// is definitely not in the set. +func (f *BloomFilter) Test(data []byte) bool { + h := baseHashes(data) + for i := uint(0); i < f.k; i++ { + if !f.b.Test(f.location(h, i)) { + return false + } + } + return true +} + +// TestString returns true if the string is in the BloomFilter, false otherwise. +// If true, the result might be a false positive. If false, the data +// is definitely not in the set. +func (f *BloomFilter) TestString(data string) bool { + return f.Test([]byte(data)) +} + +// TestLocations returns true if all locations are set in the BloomFilter, false +// otherwise. +func (f *BloomFilter) TestLocations(locs []uint64) bool { + for i := 0; i < len(locs); i++ { + if !f.b.Test(uint(locs[i] % uint64(f.m))) { + return false + } + } + return true +} + +// TestAndAdd is the equivalent to calling Test(data) then Add(data). +// Returns the result of Test. +func (f *BloomFilter) TestAndAdd(data []byte) bool { + present := true + h := baseHashes(data) + for i := uint(0); i < f.k; i++ { + l := f.location(h, i) + if !f.b.Test(l) { + present = false + } + f.b.Set(l) + } + return present +} + +// TestAndAddString is the equivalent to calling Test(string) then Add(string). +// Returns the result of Test. +func (f *BloomFilter) TestAndAddString(data string) bool { + return f.TestAndAdd([]byte(data)) +} + +// ClearAll clears all the data in a Bloom filter, removing all keys +func (f *BloomFilter) ClearAll() *BloomFilter { + f.b.ClearAll() + return f +} + +// EstimateFalsePositiveRate returns, for a BloomFilter with a estimate of m bits +// and k hash functions, what the false positive rate will be +// while storing n entries; runs 100,000 tests. This is an empirical +// test using integers as keys. As a side-effect, it clears the BloomFilter. +func (f *BloomFilter) EstimateFalsePositiveRate(n uint) (fpRate float64) { + rounds := uint32(100000) + f.ClearAll() + n1 := make([]byte, 4) + for i := uint32(0); i < uint32(n); i++ { + binary.BigEndian.PutUint32(n1, i) + f.Add(n1) + } + fp := 0 + // test for number of rounds + for i := uint32(0); i < rounds; i++ { + binary.BigEndian.PutUint32(n1, i+uint32(n)+1) + if f.Test(n1) { + //fmt.Printf("%v failed.\n", i+uint32(n)+1) + fp++ + } + } + fpRate = float64(fp) / (float64(rounds)) + f.ClearAll() + return +} + +// bloomFilterJSON is an unexported type for marshaling/unmarshaling BloomFilter struct. +type bloomFilterJSON struct { + M uint `json:"m"` + K uint `json:"k"` + B *bitset.BitSet `json:"b"` +} + +// MarshalJSON implements json.Marshaler interface. +func (f *BloomFilter) MarshalJSON() ([]byte, error) { + return json.Marshal(bloomFilterJSON{f.m, f.k, f.b}) +} + +// UnmarshalJSON implements json.Unmarshaler interface. +func (f *BloomFilter) UnmarshalJSON(data []byte) error { + var j bloomFilterJSON + err := json.Unmarshal(data, &j) + if err != nil { + return err + } + f.m = j.M + f.k = j.K + f.b = j.B + return nil +} + +// WriteTo writes a binary representation of the BloomFilter to an i/o stream. +// It returns the number of bytes written. +func (f *BloomFilter) WriteTo(stream io.Writer) (int64, error) { + err := binary.Write(stream, binary.BigEndian, uint64(f.m)) + if err != nil { + return 0, err + } + err = binary.Write(stream, binary.BigEndian, uint64(f.k)) + if err != nil { + return 0, err + } + numBytes, err := f.b.WriteTo(stream) + return numBytes + int64(2*binary.Size(uint64(0))), err +} + +// ReadFrom reads a binary representation of the BloomFilter (such as might +// have been written by WriteTo()) from an i/o stream. It returns the number +// of bytes read. +func (f *BloomFilter) ReadFrom(stream io.Reader) (int64, error) { + var m, k uint64 + err := binary.Read(stream, binary.BigEndian, &m) + if err != nil { + return 0, err + } + err = binary.Read(stream, binary.BigEndian, &k) + if err != nil { + return 0, err + } + b := &bitset.BitSet{} + numBytes, err := b.ReadFrom(stream) + if err != nil { + return 0, err + } + f.m = uint(m) + f.k = uint(k) + f.b = b + return numBytes + int64(2*binary.Size(uint64(0))), nil +} + +// GobEncode implements gob.GobEncoder interface. +func (f *BloomFilter) GobEncode() ([]byte, error) { + var buf bytes.Buffer + _, err := f.WriteTo(&buf) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// GobDecode implements gob.GobDecoder interface. +func (f *BloomFilter) GobDecode(data []byte) error { + buf := bytes.NewBuffer(data) + _, err := f.ReadFrom(buf) + + return err +} + +// Equal tests for the equality of two Bloom filters +func (f *BloomFilter) Equal(g *BloomFilter) bool { + return f.m == g.m && f.k == g.k && f.b.Equal(g.b) +} + +// Locations returns a list of hash locations representing a data item. +func Locations(data []byte, k uint) []uint64 { + locs := make([]uint64, k) + + // calculate locations + h := baseHashes(data) + for i := uint(0); i < k; i++ { + locs[i] = location(h, i) + } + + return locs +} diff --git a/vendor/github.com/willf/bloom/bloom_test.go b/vendor/github.com/willf/bloom/bloom_test.go new file mode 100644 index 0000000000..151548ab2a --- /dev/null +++ b/vendor/github.com/willf/bloom/bloom_test.go @@ -0,0 +1,597 @@ +package bloom + +import ( + "bytes" + "encoding/binary" + "encoding/gob" + "encoding/json" + "math" + "testing" +) + +// This implementation of Bloom filters is _not_ +// safe for concurrent use. Uncomment the following +// method and run go test -race +// +// func TestConcurrent(t *testing.T) { +// gmp := runtime.GOMAXPROCS(2) +// defer runtime.GOMAXPROCS(gmp) +// +// f := New(1000, 4) +// n1 := []byte("Bess") +// n2 := []byte("Jane") +// f.Add(n1) +// f.Add(n2) +// +// var wg sync.WaitGroup +// const try = 1000 +// var err1, err2 error +// +// wg.Add(1) +// go func() { +// for i := 0; i < try; i++ { +// n1b := f.Test(n1) +// if !n1b { +// err1 = fmt.Errorf("%v should be in", n1) +// break +// } +// } +// wg.Done() +// }() +// +// wg.Add(1) +// go func() { +// for i := 0; i < try; i++ { +// n2b := f.Test(n2) +// if !n2b { +// err2 = fmt.Errorf("%v should be in", n2) +// break +// } +// } +// wg.Done() +// }() +// +// wg.Wait() +// +// if err1 != nil { +// t.Fatal(err1) +// } +// if err2 != nil { +// t.Fatal(err2) +// } +// } + +func TestBasic(t *testing.T) { + f := New(1000, 4) + n1 := []byte("Bess") + n2 := []byte("Jane") + n3 := []byte("Emma") + f.Add(n1) + n3a := f.TestAndAdd(n3) + n1b := f.Test(n1) + n2b := f.Test(n2) + n3b := f.Test(n3) + if !n1b { + t.Errorf("%v should be in.", n1) + } + if n2b { + t.Errorf("%v should not be in.", n2) + } + if n3a { + t.Errorf("%v should not be in the first time we look.", n3) + } + if !n3b { + t.Errorf("%v should be in the second time we look.", n3) + } +} + +func TestBasicUint32(t *testing.T) { + f := New(1000, 4) + n1 := make([]byte, 4) + n2 := make([]byte, 4) + n3 := make([]byte, 4) + n4 := make([]byte, 4) + binary.BigEndian.PutUint32(n1, 100) + binary.BigEndian.PutUint32(n2, 101) + binary.BigEndian.PutUint32(n3, 102) + binary.BigEndian.PutUint32(n4, 103) + f.Add(n1) + n3a := f.TestAndAdd(n3) + n1b := f.Test(n1) + n2b := f.Test(n2) + n3b := f.Test(n3) + f.Test(n4) + if !n1b { + t.Errorf("%v should be in.", n1) + } + if n2b { + t.Errorf("%v should not be in.", n2) + } + if n3a { + t.Errorf("%v should not be in the first time we look.", n3) + } + if !n3b { + t.Errorf("%v should be in the second time we look.", n3) + } +} + +func TestNewWithLowNumbers(t *testing.T) { + f := New(0, 0) + if f.k != 1 { + t.Errorf("%v should be 1", f.k) + } + if f.m != 1 { + t.Errorf("%v should be 1", f.m) + } +} + +func TestString(t *testing.T) { + f := NewWithEstimates(1000, 0.001) + n1 := "Love" + n2 := "is" + n3 := "in" + n4 := "bloom" + f.AddString(n1) + n3a := f.TestAndAddString(n3) + n1b := f.TestString(n1) + n2b := f.TestString(n2) + n3b := f.TestString(n3) + f.TestString(n4) + if !n1b { + t.Errorf("%v should be in.", n1) + } + if n2b { + t.Errorf("%v should not be in.", n2) + } + if n3a { + t.Errorf("%v should not be in the first time we look.", n3) + } + if !n3b { + t.Errorf("%v should be in the second time we look.", n3) + } + +} + +func testEstimated(n uint, maxFp float64, t *testing.T) { + m, k := EstimateParameters(n, maxFp) + f := NewWithEstimates(n, maxFp) + fpRate := f.EstimateFalsePositiveRate(n) + if fpRate > 1.5*maxFp { + t.Errorf("False positive rate too high: n: %v; m: %v; k: %v; maxFp: %f; fpRate: %f, fpRate/maxFp: %f", n, m, k, maxFp, fpRate, fpRate/maxFp) + } +} + +func TestEstimated1000_0001(t *testing.T) { testEstimated(1000, 0.000100, t) } +func TestEstimated10000_0001(t *testing.T) { testEstimated(10000, 0.000100, t) } +func TestEstimated100000_0001(t *testing.T) { testEstimated(100000, 0.000100, t) } + +func TestEstimated1000_001(t *testing.T) { testEstimated(1000, 0.001000, t) } +func TestEstimated10000_001(t *testing.T) { testEstimated(10000, 0.001000, t) } +func TestEstimated100000_001(t *testing.T) { testEstimated(100000, 0.001000, t) } + +func TestEstimated1000_01(t *testing.T) { testEstimated(1000, 0.010000, t) } +func TestEstimated10000_01(t *testing.T) { testEstimated(10000, 0.010000, t) } +func TestEstimated100000_01(t *testing.T) { testEstimated(100000, 0.010000, t) } + +func min(a, b uint) uint { + if a < b { + return a + } + return b +} + +// The following function courtesy of Nick @turgon +// This helper function ranges over the input data, applying the hashing +// which returns the bit locations to set in the filter. +// For each location, increment a counter for that bit address. +// +// If the Bloom Filter's location() method distributes locations uniformly +// at random, a property it should inherit from its hash function, then +// each bit location in the filter should end up with roughly the same +// number of hits. Importantly, the value of k should not matter. +// +// Once the results are collected, we can run a chi squared goodness of fit +// test, comparing the result histogram with the uniform distribition. +// This yields a test statistic with degrees-of-freedom of m-1. +func chiTestBloom(m, k, rounds uint, elements [][]byte) (succeeds bool) { + f := New(m, k) + results := make([]uint, m) + chi := make([]float64, m) + + for _, data := range elements { + h := baseHashes(data) + for i := uint(0); i < f.k; i++ { + results[f.location(h, i)]++ + } + } + + // Each element of results should contain the same value: k * rounds / m. + // Let's run a chi-square goodness of fit and see how it fares. + var chiStatistic float64 + e := float64(k*rounds) / float64(m) + for i := uint(0); i < m; i++ { + chi[i] = math.Pow(float64(results[i])-e, 2.0) / e + chiStatistic += chi[i] + } + + // this tests at significant level 0.005 up to 20 degrees of freedom + table := [20]float64{ + 7.879, 10.597, 12.838, 14.86, 16.75, 18.548, 20.278, + 21.955, 23.589, 25.188, 26.757, 28.3, 29.819, 31.319, 32.801, 34.267, + 35.718, 37.156, 38.582, 39.997} + df := min(m-1, 20) + + succeeds = table[df-1] > chiStatistic + return + +} + +func TestLocation(t *testing.T) { + var m, k, rounds uint + + m = 8 + k = 3 + + rounds = 100000 // 15000000 + + elements := make([][]byte, rounds) + + for x := uint(0); x < rounds; x++ { + ctrlist := make([]uint8, 4) + ctrlist[0] = uint8(x) + ctrlist[1] = uint8(x >> 8) + ctrlist[2] = uint8(x >> 16) + ctrlist[3] = uint8(x >> 24) + data := []byte(ctrlist) + elements[x] = data + } + + succeeds := chiTestBloom(m, k, rounds, elements) + if !succeeds { + t.Error("random assignment is too unrandom") + } + +} + +func TestCap(t *testing.T) { + f := New(1000, 4) + if f.Cap() != f.m { + t.Error("not accessing Cap() correctly") + } +} + +func TestK(t *testing.T) { + f := New(1000, 4) + if f.K() != f.k { + t.Error("not accessing K() correctly") + } +} + +func TestMarshalUnmarshalJSON(t *testing.T) { + f := New(1000, 4) + data, err := json.Marshal(f) + if err != nil { + t.Fatal(err.Error()) + } + + var g BloomFilter + err = json.Unmarshal(data, &g) + if err != nil { + t.Fatal(err.Error()) + } + if g.m != f.m { + t.Error("invalid m value") + } + if g.k != f.k { + t.Error("invalid k value") + } + if g.b == nil { + t.Fatal("bitset is nil") + } + if !g.b.Equal(f.b) { + t.Error("bitsets are not equal") + } +} + +func TestUnmarshalInvalidJSON(t *testing.T) { + data := []byte("{invalid}") + + var g BloomFilter + err := g.UnmarshalJSON(data) + if err == nil { + t.Error("expected error while unmarshalling invalid data") + } +} + +func TestWriteToReadFrom(t *testing.T) { + var b bytes.Buffer + f := New(1000, 4) + _, err := f.WriteTo(&b) + if err != nil { + t.Fatal(err) + } + + g := New(1000, 1) + _, err = g.ReadFrom(&b) + if err != nil { + t.Fatal(err) + } + if g.m != f.m { + t.Error("invalid m value") + } + if g.k != f.k { + t.Error("invalid k value") + } + if g.b == nil { + t.Fatal("bitset is nil") + } + if !g.b.Equal(f.b) { + t.Error("bitsets are not equal") + } + + g.Test([]byte("")) +} + +func TestReadWriteBinary(t *testing.T) { + f := New(1000, 4) + var buf bytes.Buffer + bytesWritten, err := f.WriteTo(&buf) + if err != nil { + t.Fatal(err.Error()) + } + if bytesWritten != int64(buf.Len()) { + t.Errorf("incorrect write length %d != %d", bytesWritten, buf.Len()) + } + + var g BloomFilter + bytesRead, err := g.ReadFrom(&buf) + if err != nil { + t.Fatal(err.Error()) + } + if bytesRead != bytesWritten { + t.Errorf("read unexpected number of bytes %d != %d", bytesRead, bytesWritten) + } + if g.m != f.m { + t.Error("invalid m value") + } + if g.k != f.k { + t.Error("invalid k value") + } + if g.b == nil { + t.Fatal("bitset is nil") + } + if !g.b.Equal(f.b) { + t.Error("bitsets are not equal") + } +} + +func TestEncodeDecodeGob(t *testing.T) { + f := New(1000, 4) + f.Add([]byte("one")) + f.Add([]byte("two")) + f.Add([]byte("three")) + var buf bytes.Buffer + err := gob.NewEncoder(&buf).Encode(f) + if err != nil { + t.Fatal(err.Error()) + } + + var g BloomFilter + err = gob.NewDecoder(&buf).Decode(&g) + if err != nil { + t.Fatal(err.Error()) + } + if g.m != f.m { + t.Error("invalid m value") + } + if g.k != f.k { + t.Error("invalid k value") + } + if g.b == nil { + t.Fatal("bitset is nil") + } + if !g.b.Equal(f.b) { + t.Error("bitsets are not equal") + } + if !g.Test([]byte("three")) { + t.Errorf("missing value 'three'") + } + if !g.Test([]byte("two")) { + t.Errorf("missing value 'two'") + } + if !g.Test([]byte("one")) { + t.Errorf("missing value 'one'") + } +} + +func TestEqual(t *testing.T) { + f := New(1000, 4) + f1 := New(1000, 4) + g := New(1000, 20) + h := New(10, 20) + n1 := []byte("Bess") + f1.Add(n1) + if !f.Equal(f) { + t.Errorf("%v should be equal to itself", f) + } + if f.Equal(f1) { + t.Errorf("%v should not be equal to %v", f, f1) + } + if f.Equal(g) { + t.Errorf("%v should not be equal to %v", f, g) + } + if f.Equal(h) { + t.Errorf("%v should not be equal to %v", f, h) + } +} + +func BenchmarkEstimated(b *testing.B) { + for n := uint(100000); n <= 100000; n *= 10 { + for fp := 0.1; fp >= 0.0001; fp /= 10.0 { + f := NewWithEstimates(n, fp) + f.EstimateFalsePositiveRate(n) + } + } +} + +func BenchmarkSeparateTestAndAdd(b *testing.B) { + f := NewWithEstimates(uint(b.N), 0.0001) + key := make([]byte, 100) + b.ResetTimer() + for i := 0; i < b.N; i++ { + binary.BigEndian.PutUint32(key, uint32(i)) + f.Test(key) + f.Add(key) + } +} + +func BenchmarkCombinedTestAndAdd(b *testing.B) { + f := NewWithEstimates(uint(b.N), 0.0001) + key := make([]byte, 100) + b.ResetTimer() + for i := 0; i < b.N; i++ { + binary.BigEndian.PutUint32(key, uint32(i)) + f.TestAndAdd(key) + } +} + +func TestMerge(t *testing.T) { + f := New(1000, 4) + n1 := []byte("f") + f.Add(n1) + + g := New(1000, 4) + n2 := []byte("g") + g.Add(n2) + + h := New(999, 4) + n3 := []byte("h") + h.Add(n3) + + j := New(1000, 5) + n4 := []byte("j") + j.Add(n4) + + err := f.Merge(g) + if err != nil { + t.Errorf("There should be no error when merging two similar filters") + } + + err = f.Merge(h) + if err == nil { + t.Errorf("There should be an error when merging filters with mismatched m") + } + + err = f.Merge(j) + if err == nil { + t.Errorf("There should be an error when merging filters with mismatched k") + } + + n2b := f.Test(n2) + if !n2b { + t.Errorf("The value doesn't exist after a valid merge") + } + + n3b := f.Test(n3) + if n3b { + t.Errorf("The value exists after an invalid merge") + } + + n4b := f.Test(n4) + if n4b { + t.Errorf("The value exists after an invalid merge") + } +} + +func TestCopy(t *testing.T) { + f := New(1000, 4) + n1 := []byte("f") + f.Add(n1) + + // copy here instead of New + g := f.Copy() + n2 := []byte("g") + g.Add(n2) + + n1fb := f.Test(n1) + if !n1fb { + t.Errorf("The value doesn't exist in original after making a copy") + } + + n1gb := g.Test(n1) + if !n1gb { + t.Errorf("The value doesn't exist in the copy") + } + + n2fb := f.Test(n2) + if n2fb { + t.Errorf("The value exists in the original, it should only exist in copy") + } + + n2gb := g.Test(n2) + if !n2gb { + t.Errorf("The value doesn't exist in copy after Add()") + } +} + +func TestFrom(t *testing.T) { + var ( + k = uint(5) + data = make([]uint64, 10) + test = []byte("test") + ) + + bf := From(data, k) + if bf.K() != k { + t.Errorf("Constant k does not match the expected value") + } + + if bf.Cap() != uint(len(data)*64) { + t.Errorf("Capacity does not match the expected value") + } + + if bf.Test(test) { + t.Errorf("Bloom filter should not contain the value") + } + + bf.Add(test) + if !bf.Test(test) { + t.Errorf("Bloom filter should contain the value") + } + + // create a new Bloom filter from an existing (populated) data slice. + bf = From(data, k) + if !bf.Test(test) { + t.Errorf("Bloom filter should contain the value") + } +} + +func TestTestLocations(t *testing.T) { + f := NewWithEstimates(1000, 0.001) + n1 := []byte("Love") + n2 := []byte("is") + n3 := []byte("in") + n4 := []byte("bloom") + f.Add(n1) + n3a := f.TestLocations(Locations(n3, f.K())) + f.Add(n3) + n1b := f.TestLocations(Locations(n1, f.K())) + n2b := f.TestLocations(Locations(n2, f.K())) + n3b := f.TestLocations(Locations(n3, f.K())) + n4b := f.TestLocations(Locations(n4, f.K())) + if !n1b { + t.Errorf("%v should be in.", n1) + } + if n2b { + t.Errorf("%v should not be in.", n2) + } + if n3a { + t.Errorf("%v should not be in the first time we look.", n3) + } + if !n3b { + t.Errorf("%v should be in the second time we look.", n3) + } + if n4b { + t.Errorf("%v should be in.", n4) + } +}