diff --git a/qkc/serialize/bytebuffer.go b/qkc/serialize/bytebuffer.go new file mode 100644 index 0000000000..3356747882 --- /dev/null +++ b/qkc/serialize/bytebuffer.go @@ -0,0 +1,122 @@ +// Ported verbatim from github.com/QuarkChain/goquarkchain/serialize (byte-compatible). + +package serialize + +import ( + "encoding/binary" + "fmt" + "math" +) + +type ByteBuffer struct { + data *[]byte + position int + size int +} + +func NewByteBuffer(bytes []byte) *ByteBuffer { + bb := ByteBuffer{&bytes, 0, len(bytes)} + return &bb +} + +func (bb *ByteBuffer) GetOffset() int { + return bb.position +} + +func (bb *ByteBuffer) getBytes(size int) ([]byte, error) { + if size > bb.size-bb.position { + return nil, fmt.Errorf("deser: buffer is shorter than expected") + } + + bytes := (*bb.data)[bb.position : bb.position+size] + bb.position += size + return bytes, nil +} + +func (bb *ByteBuffer) GetUInt8() (uint8, error) { + bytes, err := bb.getBytes(1) + if err != nil { + return 0, err + } + + return uint8(bytes[0]), nil +} + +func (bb *ByteBuffer) GetUInt16() (uint16, error) { + bytes, err := bb.getBytes(2) + if err != nil { + return 0, err + } + + return binary.BigEndian.Uint16(bytes), nil +} + +func (bb *ByteBuffer) GetUInt32() (uint32, error) { + bytes, err := bb.getBytes(4) + if err != nil { + return 0, err + } + + return binary.BigEndian.Uint32(bytes), nil +} + +func (bb *ByteBuffer) GetUInt64() (uint64, error) { + bytes, err := bb.getBytes(8) + if err != nil { + return 0, err + } + + return binary.BigEndian.Uint64(bytes), nil +} + +func (bb *ByteBuffer) getLen(byteSize int) (int, error) { + if byteSize < 1 { + return 0, fmt.Errorf("deser: bytesize in GetVarBytes should larger than 0") + } + + b, err := bb.getBytes(byteSize) + if err != nil { + return 0, err + } + + // Accumulate in uint64 and bound the result. Using a signed int with an + // unbounded left shift let a high length-prefix byte land in the sign bit + // once the prefix reached the platform int width (4 bytes on 32-bit, 8 on + // 64-bit), silently yielding a NEGATIVE length. A negative length slips past + // downstream "len <= remaining" checks and then panics (slice bounds / + // reflect.MakeSlice: negative len) on attacker-controlled input. Reject any + // prefix that overflows uint64 or does not fit a non-negative int. Hardening + // divergence from goquarkchain; it does not change any representable length. + var size uint64 = 0 + for i := 0; i < byteSize; i++ { + if size > math.MaxUint64>>8 { + return 0, fmt.Errorf("deser: length prefix of %d bytes overflows", byteSize) + } + size = (size << 8) | uint64(b[i]) + } + if size > uint64(math.MaxInt) { + return 0, fmt.Errorf("deser: length prefix %d exceeds maximum %d", size, uint64(math.MaxInt)) + } + + return int(size), nil +} + +func (bb *ByteBuffer) GetVarBytes(byteSizeOfSliceLen int) ([]byte, error) { + size, err := bb.getLen(byteSizeOfSliceLen) + if err != nil { + return nil, err + } + + bs, err := bb.getBytes(size) + if err != nil { + return nil, err + } + + bytes := make([]byte, size, size) + copy(bytes, bs) + return bytes, nil +} + +func (bb *ByteBuffer) Remaining() int { + return bb.size - bb.position +} diff --git a/qkc/serialize/bytebuffer_getlen_test.go b/qkc/serialize/bytebuffer_getlen_test.go new file mode 100644 index 0000000000..082ab6d064 --- /dev/null +++ b/qkc/serialize/bytebuffer_getlen_test.go @@ -0,0 +1,47 @@ +package serialize + +import "testing" + +// TestGetLenRejectsOverflow pins that getLen never returns a negative or wrapped +// length. A length prefix at/above the platform int width with the high bit set +// previously overflowed the signed accumulator into a negative value, which slid +// past downstream "len <= remaining" checks and panicked (reflect.MakeSlice: +// negative len) on attacker-controlled input. +func TestGetLenRejectsOverflow(t *testing.T) { + // 8-byte prefix, high bit set: > MaxInt on every platform -> error. + bb := NewByteBuffer([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}) + if n, err := bb.getLen(8); err == nil { + t.Fatalf("expected an error for an over-large 8-byte length prefix, got %d", n) + } + + // A wide (10-byte) prefix encoding a small value still decodes: leading + // zeros must not be mistaken for overflow. + small := NewByteBuffer([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0x12, 0x34}) + if n, err := small.getLen(10); err != nil || n != 0x1234 { + t.Fatalf("small value with wide prefix: n=%d err=%v, want 4660", n, err) + } + + // getLen must never return a negative length for any input/width. + for _, w := range []int{1, 2, 4, 8} { + b := make([]byte, w) + for i := range b { + b[i] = 0xFF + } + if n, err := NewByteBuffer(b).getLen(w); err == nil && n < 0 { + t.Fatalf("getLen(%d) returned negative length %d", w, n) + } + } +} + +// TestDeserializeWideLenPrefixNoPanic ensures an attacker-controlled over-wide +// slice length prefix yields an error, not a panic. +func TestDeserializeWideLenPrefixNoPanic(t *testing.T) { + type wide struct { + V []uint32 `bytesizeofslicelen:"8"` + } + var out wide + err := Deserialize(NewByteBuffer([]byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}), &out) + if err == nil { + t.Fatal("expected an error for an over-large length prefix") + } +} diff --git a/qkc/serialize/deserializer.go b/qkc/serialize/deserializer.go new file mode 100644 index 0000000000..85ef8e2abc --- /dev/null +++ b/qkc/serialize/deserializer.go @@ -0,0 +1,307 @@ +// Ported verbatim from github.com/QuarkChain/goquarkchain/serialize (byte-compatible). + +package serialize + +import ( + "errors" + "fmt" + "math/big" + "reflect" +) + +var ( + errNoPointer = errors.New("deser: interface given to Deserialize must be a pointer") + errDeserializeIntoNil = errors.New("deser: pointer given to Deserialize must not be nil") +) + +func Deserialize(bb *ByteBuffer, val interface{}) error { + return DeserializeWithTags(bb, val, Tags{ByteSizeOfSliceLen: 1}) +} + +func DeserializeWithTags(bb *ByteBuffer, val interface{}, ts Tags) error { + if val == nil { + return errDeserializeIntoNil + } + + rval := reflect.ValueOf(val) + rtyp := rval.Type() + if rtyp.Kind() != reflect.Ptr { + return errNoPointer + } + if rval.IsNil() { + return errDeserializeIntoNil + } + + info, err := cachedTypeInfo(rtyp.Elem()) + if err != nil { + return err + } + + err = info.deserializer(bb, rval.Elem(), ts) + return err +} + +func DeserializeFromBytes(b []byte, val interface{}) error { + bb := NewByteBuffer(b) + if err := Deserialize(bb, val); err != nil { + return err + } + // Reject inputs that decode successfully but leave bytes unconsumed: valid + // data followed by trailing garbage must not be silently accepted. (The + // lower-level Deserialize stays permissive so callers can stream multiple + // values from one buffer.) + if bb.Remaining() != 0 { + return fmt.Errorf("deser: %d trailing byte(s) after decoding", bb.Remaining()) + } + return nil +} + +func makeDeserializer(typ reflect.Type) (deserializer, error) { + kind := typ.Kind() + switch { + //check Ptr first and add optional byte output if ts is nilok, + //then get serializer for typ.Elem() which is not a ptr + case kind == reflect.Ptr: + return deserializePtr, nil + case kind != reflect.Ptr && reflect.PtrTo(typ).Implements(serializableInterface): + return deserializeSerializableInterface, nil + case typ.AssignableTo(bigInt): + return deserializeBigIntNoPtr, nil + case isUint(kind): + return deserializeUint, nil + case kind == reflect.Bool: + return deserializeBool, nil + case kind == reflect.String: + return deserializeString, nil + case kind == reflect.Slice && isByte(typ.Elem()): + return deserializeByteSlice, nil + case kind == reflect.Array && isByte(typ.Elem()): + return deserializeByteArray, nil + case kind == reflect.Slice || kind == reflect.Array: + return deserializeList, nil + case kind == reflect.Struct: + return deserializeStruct, nil + default: + return nil, fmt.Errorf("type %v is not serializable", typ) + } +} + +func deserializeSerializableInterface(bb *ByteBuffer, val reflect.Value, ts Tags) error { + return val.Addr().Interface().(Serializable).Deserialize(bb) +} + +func deserializeUint(bb *ByteBuffer, val reflect.Value, ts Tags) error { + kind := val.Type().Kind() + var bytes []byte + var err error + switch { + case kind > reflect.Uint && kind <= reflect.Uintptr: + bytes, err = bb.getBytes(val.Type().Bits() / 8) + break + case kind == reflect.Uint: + bytes, err = bb.GetVarBytes(1) + break + default: + err = fmt.Errorf("deser: invalid Uint type: %s", val.Type().Name()) + break + } + + if err == nil { + // Reject encodings wider than the destination. The variable-length + // reflect.Uint path (GetVarBytes, 1-byte length prefix) can otherwise + // deliver more bytes than the uint width, overflowing the uint64 + // accumulator below and silently truncating the value. Fixed-width kinds + // always read exactly Bits()/8 bytes, so this never triggers for them. + if len(bytes) > val.Type().Bits()/8 { + return fmt.Errorf("deser: uint encoding too long: %d bytes exceeds the %d-bit destination width", len(bytes), val.Type().Bits()) + } + var ui uint64 = 0 + for i := 0; i < len(bytes); i++ { + ui = ui<<8 | uint64(bytes[i]) + } + val.SetUint(ui) + } + + return err +} + +func deserializeFixSizeBigUint(bb *ByteBuffer, val *big.Int, size int) error { + bytes, err := bb.getBytes(size) + if err == nil { + val.SetBytes(bytes) + } + + return err +} + +func deserializeBigIntNoPtr(bb *ByteBuffer, val reflect.Value, ts Tags) error { + return deserializeBigInt(bb, val.Addr()) +} + +func deserializeBigInt(bb *ByteBuffer, val reflect.Value) error { + bytes, err := bb.GetVarBytes(1) + if err != nil { + return err + } + + i := val.Interface().(*big.Int) + if i == nil { + i = new(big.Int) + val.Set(reflect.ValueOf(i)) + } + + i.SetBytes(bytes) + return nil +} + +func deserializeBool(bb *ByteBuffer, val reflect.Value, ts Tags) error { + b, err := bb.getBytes(1) + if err == nil { + switch b[0] { + case 0x00: + val.SetBool(false) + case 0x01: + val.SetBool(true) + default: + err = fmt.Errorf("deser: invalid boolean value: %d", b[0]) + } + } + + return err +} + +// FixedSizeBytes +func deserializeByteArray(bb *ByteBuffer, val reflect.Value, ts Tags) error { + if val.Kind() != reflect.Array { + return fmt.Errorf("deser: invalid byte array type: %s", val.Kind()) + } + if val.Type().Elem().Kind() != reflect.Uint8 { + return fmt.Errorf("deser: invalid byte array type: [%d]%s", val.Len(), val.Kind()) + } + + bytes, err := bb.getBytes(val.Len()) + if err == nil { + reflect.Copy(val, reflect.ValueOf(bytes)) + } + + return err +} + +// deserializePrependedSizeBytes +func deserializeByteSlice(bb *ByteBuffer, val reflect.Value, ts Tags) error { + bytes, err := bb.GetVarBytes(ts.ByteSizeOfSliceLen) + if err == nil { + val.SetBytes(bytes) + } + + return err +} + +func deserializeList(bb *ByteBuffer, val reflect.Value, ts Tags) error { + typeinfo, err := cachedTypeInfo(val.Type().Elem()) + if err != nil { + return err + } + + var vlen int = 0 + if val.Kind() == reflect.Slice { + vlen, err = bb.getLen(ts.ByteSizeOfSliceLen) + if err != nil { + return err + } + + // Bound the element count by the bytes left in the buffer before + // allocating. Every element of a registrable slice consumes at least one + // byte — zero-byte element types are rejected at registration (see + // encodesZeroBytes in genTypeInfo) — so a vlen larger than Remaining() can + // only come from a malformed/malicious input (it would fail below when + // reading elements anyway). Without this guard a tiny packet claiming a + // huge count (up to ~4B with bytesizeofslicelen:4) forces a massive + // MakeSlice allocation — an OOM/DoS vector, since this codec decodes + // blocks/headers straight from peers. This is an intentional hardening + // divergence from goquarkchain; it does not change the decoding of any + // valid input. + if vlen > bb.Remaining() { + return fmt.Errorf("deser: list length %d exceeds remaining buffer %d", vlen, bb.Remaining()) + } + + newv := reflect.MakeSlice(val.Type(), vlen, vlen) + reflect.Copy(newv, val) + val.Set(newv) + } else if val.Kind() == reflect.Array { + vlen = val.Len() + } + + for i := 0; i < vlen; i++ { + if err := typeinfo.deserializer(bb, val.Index(i), Tags{ByteSizeOfSliceLen: 1}); err != nil { + return err + } + } + + return nil +} + +func deserializeStruct(bb *ByteBuffer, val reflect.Value, ts Tags) error { + fields, err := structFields(val.Type()) + if err != nil { + return err + } + + for _, f := range fields { + err := f.info.deserializer(bb, val.Field(f.index), f.tags) + if err != nil { + return fmt.Errorf("%s for %v%s", err.Error(), val.Type(), "."+val.Type().Field(f.index).Name) + } + } + + return nil +} + +func deserializeString(bb *ByteBuffer, val reflect.Value, ts Tags) error { + b, err := bb.GetVarBytes(4) + if err != nil { + return err + } + + val.SetString(string(b)) + return nil +} + +func deserializePtr(bb *ByteBuffer, val reflect.Value, ts Tags) error { + typ := val.Type() + typeinfo, err := cachedTypeInfo(typ.Elem()) + if err != nil { + return err + } + + if ts.NilOK { + b, err := bb.GetUInt8() + if err != nil { + return err + } + + if b == 0 { + // set the pointer to nil. + val.Set(reflect.Zero(typ)) + return nil + } + // The presence marker is written as exactly 0 or 1 (see serializePtr), so + // any other value is a non-canonical encoding and must be rejected — + // otherwise e.g. 0x05 would decode identically to 0x01. + if b != 1 { + return fmt.Errorf("deser: invalid nil-pointer presence marker %d (must be 0 or 1)", b) + } + } + + newval := val + if val.IsNil() { + newval = reflect.New(typ.Elem()) + } + + err = typeinfo.deserializer(bb, newval.Elem(), ts) + if err == nil { + val.Set(newval) + } + + return err +} diff --git a/qkc/serialize/deserializer_bound_test.go b/qkc/serialize/deserializer_bound_test.go new file mode 100644 index 0000000000..02205f8a32 --- /dev/null +++ b/qkc/serialize/deserializer_bound_test.go @@ -0,0 +1,35 @@ +package serialize + +import "testing" + +// TestDeserializeListBound guards the OOM/DoS vector where a tiny packet claims a +// huge list length: deserializeList must reject a length larger than the +// remaining buffer instead of pre-allocating it. +func TestDeserializeListBound(t *testing.T) { + type bigList struct { + V []uint32 `bytesizeofslicelen:"4"` + } + // 4-byte length prefix = 0xFFFFFFFF (~4 billion), followed by no element bytes. + input := unhex("FFFFFFFF") + var out bigList + if err := Deserialize(NewByteBuffer(input), &out); err == nil { + t.Fatal("expected an error for an over-large list length, got nil (would over-allocate ~4B elements)") + } +} + +// TestDeserializeListValidStillDecodes confirms the bound does not reject valid +// input where length <= remaining bytes. +func TestDeserializeListValidStillDecodes(t *testing.T) { + type list struct { + V []uint32 `bytesizeofslicelen:"4"` + } + // len = 2 (4 bytes), then two uint32s (8 bytes): 0x00000007, 0x00000009. + input := unhex("000000020000000700000009") + var out list + if err := Deserialize(NewByteBuffer(input), &out); err != nil { + t.Fatalf("valid list rejected: %v", err) + } + if len(out.V) != 2 || out.V[0] != 7 || out.V[1] != 9 { + t.Fatalf("bad decode: %v", out.V) + } +} diff --git a/qkc/serialize/deserializer_nilmarker_test.go b/qkc/serialize/deserializer_nilmarker_test.go new file mode 100644 index 0000000000..534d3f5e58 --- /dev/null +++ b/qkc/serialize/deserializer_nilmarker_test.go @@ -0,0 +1,36 @@ +package serialize + +import "testing" + +type withNilPtr struct { + P *uint32 `ser:"nil"` +} + +// TestDeserializeNilMarkerCanonical checks that the ser:"nil" presence marker +// only accepts 0 or 1. The serializer writes exactly those two values, so any +// other byte is a non-canonical encoding and must be rejected. +func TestDeserializeNilMarkerCanonical(t *testing.T) { + // marker 0x00 -> nil pointer. + var nilCase withNilPtr + if err := Deserialize(NewByteBuffer([]byte{0x00}), &nilCase); err != nil { + t.Fatalf("marker 0 should decode to nil: %v", err) + } + if nilCase.P != nil { + t.Fatalf("marker 0 should leave pointer nil, got %v", *nilCase.P) + } + + // marker 0x01 followed by a 4-byte uint32 -> present. + var present withNilPtr + if err := Deserialize(NewByteBuffer([]byte{0x01, 0x00, 0x00, 0x00, 0x07}), &present); err != nil { + t.Fatalf("marker 1 should decode the value: %v", err) + } + if present.P == nil || *present.P != 7 { + t.Fatalf("marker 1 should decode value 7, got %v", present.P) + } + + // marker 0x05 must be rejected (non-canonical "present" byte). + var bad withNilPtr + if err := Deserialize(NewByteBuffer([]byte{0x05, 0x00, 0x00, 0x00, 0x07}), &bad); err == nil { + t.Fatal("expected an error for a non-0/1 presence marker") + } +} diff --git a/qkc/serialize/deserializer_test.go b/qkc/serialize/deserializer_test.go new file mode 100644 index 0000000000..d3d602bf02 --- /dev/null +++ b/qkc/serialize/deserializer_test.go @@ -0,0 +1,301 @@ +// Ported verbatim from github.com/QuarkChain/goquarkchain/serialize (byte-compatible). + +package serialize + +import ( + "encoding/hex" + "fmt" + "math/big" + "reflect" + "strings" + "testing" +) + +type testDataForDeserialize struct { + input string + ptr interface{} + value interface{} + error string +} + +type simplestruct struct { + A uint + B string +} + +var ( + veryBigInt = big.NewInt(0).Add( + big.NewInt(0).Lsh(big.NewInt(0xFFFFFFFFFFFFFF), 16), + big.NewInt(0xFFFF), + ) +) + +type hasIgnoredField struct { + A uint + B uint `ser:"-"` + C uint32 +} + +var deserdata = []testDataForDeserialize{ + // booleans + {input: "01", ptr: new(bool), value: true}, + {input: "00", ptr: new(bool), value: false}, + {input: "02", ptr: new(bool), error: "deser: invalid boolean value: 2"}, + + // integers + + {input: "00", ptr: new(uint8), value: uint8(0)}, + {input: "05", ptr: new(uint8), value: uint8(5)}, + {input: "0000", ptr: new(uint16), value: uint16(0)}, + {input: "0005", ptr: new(uint16), value: uint16(5)}, + {input: "05", ptr: new(uint16), error: "deser: buffer is shorter than expected"}, + {input: "00000000", ptr: new(uint32), value: uint32(0)}, + {input: "00000505", ptr: new(uint32), value: uint32(0x0505)}, + {input: "05050505", ptr: new(uint32), value: uint32(0x05050505)}, + {input: "0000000000000000", ptr: new(uint64), value: uint64(0)}, + {input: "0000000000000505", ptr: new(uint64), value: uint64(0x0505)}, + {input: "0505050505050505", ptr: new(uint64), value: uint64(0x0505050505050505)}, + {input: "0100", ptr: new(uint), value: uint(0)}, + {input: "020505", ptr: new(uint), value: uint(0x0505)}, + {input: "080505050505050505", ptr: new(uint), value: uint(0x0505050505050505)}, + + {input: "00000000000000000000000000000001", ptr: new(Uint128), value: *newUint128(1)}, + {input: "00000000000000000000000000000080", ptr: new(Uint128), value: *newUint128(128)}, + {input: "0000000000000000FFFFFFFFFFFFFFFF", ptr: new(Uint128), value: *newUint128(0xFFFFFFFFFFFFFFFF)}, + {input: "05", ptr: new(Uint128), error: "deser: buffer is shorter than expected"}, + {input: "0000000000000000000000000000000000000000000000000000000000000001", ptr: new(Uint256), value: *newUint256(1)}, + {input: "0000000000000000000000000000000000000000000000000000000000000080", ptr: new(Uint256), value: *newUint256(128)}, + {input: "000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFF", ptr: new(Uint256), value: *newUint256(0xFFFFFFFFFFFFFFFF)}, + {input: "05", ptr: new(Uint256), error: "deser: buffer is shorter than expected"}, + + //uint slices + {input: "00", ptr: new([]uint), value: []uint{}}, + {input: "080102030405060708", ptr: new([]uint8), value: []uint8{1, 2, 3, 4, 5, 6, 7, 8}}, + {input: "080000000100000002000000030000000400000005000000060000000700000008", ptr: new([]uint32), value: []uint32{1, 2, 3, 4, 5, 6, 7, 8}}, + {input: "050102", ptr: new([]uint8), error: "deser: buffer is shorter than expected"}, + + // arrays + {input: "0102030405", ptr: new([5]uint8), value: [5]uint8{1, 2, 3, 4, 5}}, + {input: "0000000100000002000000030000000400000005", ptr: new([5]uint32), value: [5]uint32{1, 2, 3, 4, 5}}, + {input: "", ptr: new([0]uint), value: [0]uint{}}, // zero sized arrays + {input: "0102", ptr: new([5]uint8), error: "deser: buffer is shorter than expected"}, + + // byte slices + {input: "0101", ptr: new([]byte), value: []byte{1}}, + {input: "00", ptr: new([]byte), value: []byte{}}, + {input: "0D6162636465666768696A6B6C6D", ptr: new([]byte), value: []byte("abcdefghijklm")}, + {input: "0D0102", ptr: new([]byte), error: "deser: buffer is shorter than expected"}, + + // byte slices, strings + {input: "00", ptr: new([]byte), value: []byte{}}, + {input: "017E", ptr: new([]byte), value: []byte{0x7E}}, + {input: "0180", ptr: new([]byte), value: []byte{0x80}}, + {input: "03010203", ptr: new([]byte), value: []byte{1, 2, 3}}, + {input: "04010203", ptr: new([]byte), error: "deser: buffer is shorter than expected"}, + + //SerializableList interface + {input: "00000000", ptr: new(LargeBytes), value: LargeBytes{[]byte{}}}, + {input: "000000017E", ptr: new(LargeBytes), value: LargeBytes{[]byte{0x7E}}}, + {input: "0000000180", ptr: new(LargeBytes), value: LargeBytes{[]byte{0x80}}}, + {input: "00000003010203", ptr: new(LargeBytes), value: LargeBytes{[]byte{1, 2, 3}}}, + {input: "03010203", ptr: new(LargeBytes), error: "deser: buffer is shorter than expected for serialize.LargeBytes.Value"}, + + // byte arrays + {input: "00", ptr: new([0]byte), value: [0]byte{}}, + {input: "02", ptr: new([1]byte), value: [1]byte{2}}, + {input: "80", ptr: new([1]byte), value: [1]byte{128}}, + {input: "0102030405", ptr: new([5]byte), value: [5]byte{1, 2, 3, 4, 5}}, + + // strings + {input: "0000000100", ptr: new(string), value: "\000"}, + {input: "0000000D6162636465666768696A6B6C6D", ptr: new(string), value: "abcdefghijklm"}, + {input: "0D6162636465666768696A6B6C6D", ptr: new(string), error: "deser: buffer is shorter than expected"}, + + // big ints + {input: "0101", ptr: new(*big.Int), value: big.NewInt(1)}, + {input: "09FFFFFFFFFFFFFFFFFF", ptr: new(*big.Int), value: veryBigInt}, + {input: "0110", ptr: new(big.Int), value: *big.NewInt(16)}, // non-pointer also works + {input: "0210", ptr: new(big.Int), error: "deser: buffer is shorter than expected"}, + + // structs + {input: "0301020300", ptr: new(structForTest), value: newStructForTest(&[]byte{1, 2, 3}, nil)}, + {input: "030102030103040506", ptr: new(structForTest), value: newStructForTest(&[]byte{1, 2, 3}, &[]byte{4, 5, 6})}, + // To present (canonical marker 01) but its slice buffer is truncated. (The + // presence marker must be exactly 0 or 1; non-canonical markers like 03 are + // rejected — covered by TestDeserializeNilMarkerCanonical.) + {input: "0301020301040506", ptr: new(structForTest), error: "deser: buffer is shorter than expected for serialize.structForTest.To"}, + + // structs + { + input: "010500000003343434", + ptr: new(simplestruct), + value: simplestruct{5, "444"}, + }, + + // struct tag "-" + { + input: "010100000002", + ptr: new(hasIgnoredField), + value: hasIgnoredField{A: 1, C: 2}, + }, + + // pointers + {input: "0100", ptr: new(*[]byte), value: &[]byte{0}}, + {input: "0100", ptr: new(*uint), value: uintp(0)}, + {input: "0107", ptr: new(*uint), value: uintp(7)}, + {input: "0180", ptr: new(*uint), value: uintp(0x80)}, + {input: "010109", ptr: new(*[]uint), value: &[]uint{9}}, + {input: "010403030303", ptr: new(*[][]byte), value: &[][]byte{{3, 3, 3, 3}}}, + + // do not support interface{}, need to know the real type + {input: "02", ptr: new(int), error: "type int is not serializable"}, + {input: "00", ptr: new(interface{}), error: "type interface {} is not serializable"}, +} + +func uintp(i uint) *uint { return &i } + +func runTests(t *testing.T, deserialize func([]byte, interface{}) error) { + for i, test := range deserdata { + input, err := hex.DecodeString(test.input) + if err != nil { + t.Errorf("test %d: invalid hex input %q", i, test.input) + continue + } + err = deserialize(input, test.ptr) + if err != nil && test.error == "" { + t.Errorf("test %d: unexpected Deserialize error: %v\ndecoding into %T\ninput %q", + i, err, test.ptr, test.input) + continue + } + if test.error != "" && fmt.Sprint(err) != test.error { + t.Errorf("test %d: Deserialize error mismatch\ngot %v\nwant %v\ndecoding into %T\ninput %q", + i, err, test.error, test.ptr, test.input) + continue + } + deref := reflect.ValueOf(test.ptr).Elem().Interface() + if err == nil && !reflect.DeepEqual(deref, test.value) { + t.Errorf("test %d: value mismatch\ngot %#v\nwant %#v\ndecoding into %T\ninput %q", + i, deref, test.value, test.ptr, test.input) + } + } +} + +func TestDeserialize(t *testing.T) { + runTests(t, func(input []byte, into interface{}) error { + return Deserialize(NewByteBuffer(input), into) + }) +} + +func ExampleDeserialize() { + input, _ := hex.DecodeString("010a0000001400000006666F6F626172") + + type example struct { + A uint + B uint32 + private uint // private fields are Ignored + String string + } + + var s example + err := Deserialize(NewByteBuffer(input), &s) + if err != nil { + fmt.Printf("Error: %v\n", err) + } else { + fmt.Printf("Deserialized value: %#v\n", s) + } + // Output: + // Deserialized value: serialize.example{A:0xa, B:0x14, private:0x0, String:"foobar"} +} + +func ExampleDeserialize_structTagNilAndIgnore() { + // In this example, we'll use the "nil" struct tag to change + // how a pointer-typed field is deserialized. The input contains an RLP + // list of one element, an empty string. + input := []byte{0x00, 0x01, 0x03, 0x04, 0x05, 0x06} + + s := new(structForTest) + Deserialize(NewByteBuffer(input), &s) + fmt.Printf("From = %v\n", *s.From) + fmt.Printf("To = %v\n", *s.To) + + // Output: + // From = [] + // To = [4 5 6] +} + +func ExampleDeserialize_structTagNil() { + // In this example, we'll use the "nil" struct tag to change + // how a pointer-typed field is deserialized. The input contains an RLP + // list of one element, an empty string. + input := []byte{0x00} + + // This type uses the normal rules. + // The empty input string is deserialized as a pointer to an empty Go string. + var normalRules struct { + String *[]byte + } + Deserialize(NewByteBuffer(input), &normalRules) + fmt.Printf("normal: String = %v\n", *normalRules.String) + + // This type uses the struct tag. + // The empty input string is deserialized as a nil pointer. + var withEmptyOK struct { + String *[]byte `ser:"nil"` + } + Deserialize(NewByteBuffer(input), &withEmptyOK) + fmt.Printf("with nil tag: String = %v\n", withEmptyOK.String) + + // Output: + // normal: String = [] + // with nil tag: String = +} + +func BenchmarkDeserialize(b *testing.B) { + enc := encodeTestSlice(90000) + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + var s []uint + bb := NewByteBuffer(enc) + if err := Deserialize(bb, &s); err != nil { + b.Fatalf("Deserialize error: %v", err) + } + } +} + +func BenchmarkDeserializeIntSliceReuse(b *testing.B) { + enc := encodeTestSlice(100000) + b.SetBytes(int64(len(enc))) + b.ReportAllocs() + b.ResetTimer() + + var s []uint + for i := 0; i < b.N; i++ { + bb := NewByteBuffer(enc) + if err := Deserialize(bb, &s); err != nil { + b.Fatalf("Deserialize error: %v", err) + } + } +} + +func encodeTestSlice(n uint) []byte { + s := make([]uint, n) + for i := uint(0); i < n; i++ { + s[i] = i + } + b, err := SerializeToBytes(s) + if err != nil { + panic(fmt.Sprintf("encode error: %v", err)) + } + return b +} + +func unhex(str string) []byte { + b, err := hex.DecodeString(strings.Replace(str, " ", "", -1)) + if err != nil { + panic(fmt.Sprintf("invalid hex string: %q", str)) + } + return b +} diff --git a/qkc/serialize/deserializer_trailing_test.go b/qkc/serialize/deserializer_trailing_test.go new file mode 100644 index 0000000000..be1f2f00da --- /dev/null +++ b/qkc/serialize/deserializer_trailing_test.go @@ -0,0 +1,19 @@ +package serialize + +import "testing" + +// TestDeserializeFromBytesRejectsTrailing checks that valid data followed by +// trailing bytes is rejected rather than silently accepted. +func TestDeserializeFromBytesRejectsTrailing(t *testing.T) { + // uint8 consumes exactly one byte; the second byte is trailing garbage. + var v uint8 + if err := DeserializeFromBytes([]byte{0x05, 0xff}, &v); err == nil { + t.Fatal("expected an error for trailing bytes after a complete decode") + } + + // An exact input still decodes. + var ok uint8 + if err := DeserializeFromBytes([]byte{0x05}, &ok); err != nil || ok != 5 { + t.Fatalf("exact input should decode cleanly: err=%v v=%d", err, ok) + } +} diff --git a/qkc/serialize/deserializer_uintwidth_test.go b/qkc/serialize/deserializer_uintwidth_test.go new file mode 100644 index 0000000000..d48f5dc17a --- /dev/null +++ b/qkc/serialize/deserializer_uintwidth_test.go @@ -0,0 +1,38 @@ +package serialize + +import ( + "math/bits" + "testing" +) + +// TestDeserializeUintWidthBound checks that the variable-length reflect.Uint +// path rejects an encoding wider than the platform uint: such an input would +// overflow the uint64 accumulator and silently truncate the decoded value. +func TestDeserializeUintWidthBound(t *testing.T) { + width := bits.UintSize / 8 // bytes in a platform uint (4 on 32-bit, 8 on 64-bit) + + // One byte past the platform width: must be rejected. + over := make([]byte, 1+width+1) + over[0] = byte(width + 1) // 1-byte length prefix + for i := 1; i < len(over); i++ { + over[i] = 0xFF + } + var u uint + if err := Deserialize(NewByteBuffer(over), &u); err == nil { + t.Fatalf("expected an error for a %d-byte uint encoding (platform width %d bytes)", width+1, width) + } + + // Exactly the platform width (max uint) still decodes. + in := make([]byte, 1+width) + in[0] = byte(width) + for i := 1; i < len(in); i++ { + in[i] = 0xFF + } + var ok uint + if err := Deserialize(NewByteBuffer(in), &ok); err != nil { + t.Fatalf("%d-byte uint rejected: %v", width, err) + } + if ok != ^uint(0) { + t.Fatalf("expected max uint, got %d", ok) + } +} diff --git a/qkc/serialize/serializer.go b/qkc/serialize/serializer.go new file mode 100644 index 0000000000..00205a886b --- /dev/null +++ b/qkc/serialize/serializer.go @@ -0,0 +1,321 @@ +// Ported verbatim from github.com/QuarkChain/goquarkchain/serialize (byte-compatible). + +package serialize + +import ( + "encoding/binary" + "errors" + "fmt" + "math/big" + "reflect" +) + +// errSerializeNil is returned when Serialize is given an untyped nil, which +// carries no type to encode. Mirrors DeserializeWithTags rejecting a nil target +// (rather than panicking in reflect.ValueOf(nil).Type()). +var errSerializeNil = errors.New("ser: nil value given to Serialize") + +func Serialize(w *[]byte, val interface{}) error { + return SerializeWithTags(w, val, Tags{ByteSizeOfSliceLen: 1}) +} + +func SerializeWithTags(w *[]byte, val interface{}, ts Tags) error { + if val == nil { + return errSerializeNil + } + + rval := reflect.ValueOf(val) + ti, err := cachedTypeInfo(rval.Type()) + if err != nil { + return err + } + + return ti.serializer(rval, w, ts) +} + +// SerializeToBytes returns the serialize result of val. +func SerializeToBytes(val interface{}) ([]byte, error) { + w := make([]byte, 0, 512) + if err := Serialize(&w, val); err != nil { + return nil, err + } + + return w, nil +} + +func makeSerializer(typ reflect.Type) (serializer, error) { + kind := typ.Kind() + switch { + //check Ptr first and add optional byte output if ts is nilok, + //then get serializer for typ.Elem() which is not a ptr + case kind == reflect.Ptr: + return serializePtr, nil + case kind != reflect.Ptr && reflect.PtrTo(typ).Implements(serializableInterface): + return serializeSerializableInterface, nil + case typ.AssignableTo(bigInt): + return serializeBigIntNoPtr, nil + case isUint(kind): + return serializeUint, nil + case kind == reflect.Bool: + return serializeBool, nil + case kind == reflect.String: + return serializeString, nil + case kind == reflect.Slice && isByte(typ.Elem()): + return serializeByteSlice, nil + case kind == reflect.Array && isByte(typ.Elem()): + return serializeByteArray, nil + case kind == reflect.Slice || kind == reflect.Array: + return serializeList, nil + case kind == reflect.Struct: + return serializeStruct, nil + default: + return nil, fmt.Errorf("type %v is not serializable", typ) + } +} + +func serializeSerializableInterface(val reflect.Value, w *[]byte, tags Tags) error { + if !val.CanAddr() { + return fmt.Errorf("ser: unaddressable value of type %v, Serialize is pointer method", val.Type()) + } + + return val.Addr().Interface().(Serializable).Serialize(w) +} + +func prefillByteArray(size int, barray []byte) ([]byte, error) { + len := len(barray) + if len > size { + return nil, errors.New("barray len is larger then expected size") + } + if len == size { + return barray, nil + } + + bytes := make([]byte, size, size) + var startIndex = size - len + copy(bytes[startIndex:], barray) + return bytes, nil +} + +func serializeFixSizeBigUint(val *big.Int, size int, w *[]byte) error { + if val == nil { + bytes := make([]byte, size, size) + *w = append(*w, bytes...) + return nil + } + // big.Int.Bytes() returns the absolute value, so without this check a + // negative value would serialize identically to its positive counterpart + // (e.g. -1 as +1), silently changing it. Reject it, matching the + // variable-length serializeBigInt path. This is an intentional hardening + // divergence from the verbatim goquarkchain port, whose fixed-size path has + // the same silent sign loss; it does not change any valid (non-negative) + // encoding. + if val.Sign() < 0 { + return fmt.Errorf("ser: cannot serialize negative big.Int") + } + bytes, err := prefillByteArray(size, val.Bytes()) + if err == nil { + *w = append(*w, bytes...) + } + + return err +} + +func serializeBigIntNoPtr(val reflect.Value, w *[]byte, ts Tags) error { + i := val.Interface().(big.Int) + return serializeBigInt(&i, w) +} + +func serializeBigInt(i *big.Int, w *[]byte) error { + var bytes []byte + if cmp := i.Cmp(big.NewInt(0)); cmp == -1 { + return fmt.Errorf("ser: cannot serialize negative *big.Int") + } else if cmp > 0 { + bytes = i.Bytes() + } + + // The length is written as a single byte, so a value needing more than 255 + // bytes (>2040 bits) would have its length truncated by uint8() and decode + // to a different value — a non-roundtrippable encoding. Reject it. (Real + // QKC quantities are far below this; this guards the general codec.) + if len(bytes) > 255 { + return fmt.Errorf("ser: big.Int too large to serialize: %d bytes exceeds the single-byte length prefix (max 255)", len(bytes)) + } + + *w = append(*w, uint8(len(bytes))) + *w = append(*w, bytes...) + return nil +} + +func serializeBool(val reflect.Value, w *[]byte, ts Tags) error { + if val.Bool() { + *w = append(*w, 0x01) + } else { + *w = append(*w, 0x00) + } + + return nil +} + +func serializeByteArray(val reflect.Value, w *[]byte, ts Tags) error { + if !val.CanAddr() { + // Slice requires the value to be addressable. + // Make it addressable by copying. + copy := reflect.New(val.Type()).Elem() + copy.Set(val) + val = copy + } + + size := val.Len() + slice := val.Slice(0, size).Bytes() + + *w = append(*w, slice...) + return nil +} + +func writeListLen(w *[]byte, len int, byteSizeOfSliceLen int) error { + sizeBytes := make([]byte, byteSizeOfSliceLen) + for i := byteSizeOfSliceLen - 1; i >= 0 && len != 0; i-- { + sizeBytes[i] = byte(len) + len = len >> 8 + } + if len > 0 { + return errors.New("barray len is larger then expected size") + } + + *w = append(*w, sizeBytes...) + return nil +} + +// serializePrependedSizeBytes +func serializeByteSlice(val reflect.Value, w *[]byte, ts Tags) error { + err := writeListLen(w, val.Len(), ts.ByteSizeOfSliceLen) + if err != nil { + return err + } + + bytes := val.Bytes() + *w = append(*w, bytes...) + return nil +} + +// PrependedSizeListSerializer +func serializeList(val reflect.Value, w *[]byte, ts Tags) error { + typeinfo, err := cachedTypeInfo(val.Type().Elem()) + if err != nil { + return err + } + + if val.Kind() == reflect.Slice { + err = writeListLen(w, val.Len(), ts.ByteSizeOfSliceLen) + if err != nil { + return err + } + } + + vlen := val.Len() + for i := 0; i < vlen; i++ { + if err := typeinfo.serializer(val.Index(i), w, Tags{ByteSizeOfSliceLen: 1}); err != nil { + return err + } + } + + return nil +} + +func serializeStruct(val reflect.Value, w *[]byte, ts Tags) error { + fields, err := structFields(val.Type()) + if err != nil { + return err + } + + for _, f := range fields { + if err := f.info.serializer(val.Field(f.index), w, f.tags); err != nil { + return err + } + } + + return nil +} + +func SerializeStructWithout(val reflect.Value, w *[]byte, excludeList map[string]bool) error { + fields, err := structFields(val.Type()) + if err != nil { + return err + } + + for _, f := range fields { + if excludeList != nil { + if _, ok := excludeList[f.name]; ok { + continue + } + } + + if err := f.info.serializer(val.Field(f.index), w, f.tags); err != nil { + return err + } + } + + return nil +} + +func serializeString(val reflect.Value, w *[]byte, ts Tags) error { + s := val.String() + + sizeBytes := make([]byte, 4, 4) + binary.BigEndian.PutUint32(sizeBytes, uint32(val.Len())) + + *w = append(*w, sizeBytes...) + *w = append(*w, s...) + return nil +} + +func serializePtr(val reflect.Value, w *[]byte, ts Tags) error { + typ := val.Type() + typeinfo, err := cachedTypeInfo(typ.Elem()) + if err != nil { + return err + } + switch { + case val.IsNil() && ts.NilOK: + *w = append(*w, 0) + return nil + case val.IsNil() && typ.Implements(serializableInterface): + zero := reflect.New(typ.Elem()) + return typeinfo.serializer(zero.Elem(), w, ts) + case val.IsNil(): + zero := reflect.Zero(typ.Elem()) + return typeinfo.serializer(zero, w, ts) + default: + if ts.NilOK { + *w = append(*w, 1) + } + return typeinfo.serializer(val.Elem(), w, ts) + } +} + +func serializeUint(val reflect.Value, w *[]byte, ts Tags) error { + kind := val.Type().Kind() + value := val.Uint() + switch kind { + case reflect.Uint8: + *w = append(*w, byte(value)) + return nil + case reflect.Uint16: + *w = append(*w, byte(value>>8), byte(value)) + break + case reflect.Uint32: + *w = append(*w, byte(value>>24), byte(value>>16), byte(value>>8), byte(value)) + return nil + case reflect.Uint64: + *w = append(*w, byte(value>>56), byte(value>>48), byte(value>>40), byte(value>>32), + byte(value>>24), byte(value>>16), byte(value>>8), byte(value)) + return nil + case reflect.Uint: + //As Uint would be 32/64 bit, so + var bi big.Int + return serializeBigInt(bi.SetUint64(val.Uint()), w) + default: + return fmt.Errorf("ser: invalid Uint type: %s", val.Type().Name()) + } + return nil +} diff --git a/qkc/serialize/serializer_bound_test.go b/qkc/serialize/serializer_bound_test.go new file mode 100644 index 0000000000..de55ac7249 --- /dev/null +++ b/qkc/serialize/serializer_bound_test.go @@ -0,0 +1,36 @@ +package serialize + +import ( + "math/big" + "testing" +) + +// TestSerializeBigIntLengthBound guards the variable-length big.Int encoding: +// its length prefix is a single byte, so a value needing more than 255 bytes +// would truncate the prefix and decode to a different value. It must error +// instead, and values at/under the limit must still round-trip. +func TestSerializeBigIntLengthBound(t *testing.T) { + // 2^2048 needs 257 bytes (> 255): must be rejected. + tooBig := new(big.Int).Lsh(big.NewInt(1), 2048) + var w []byte + if err := Serialize(&w, tooBig); err == nil { + t.Fatal("expected an error serializing a big.Int needing >255 bytes (non-roundtrippable length prefix)") + } + + // 2^2032 needs exactly 255 bytes: the maximum that fits, must round-trip. + maxFit := new(big.Int).Lsh(big.NewInt(1), 8*254) + if got := len(maxFit.Bytes()); got != 255 { + t.Fatalf("test setup: expected 255 bytes, got %d", got) + } + w = nil + if err := Serialize(&w, maxFit); err != nil { + t.Fatalf("255-byte big.Int rejected: %v", err) + } + var back big.Int + if err := Deserialize(NewByteBuffer(w), &back); err != nil { + t.Fatalf("deserialize: %v", err) + } + if back.Cmp(maxFit) != 0 { + t.Fatalf("round-trip mismatch") + } +} diff --git a/qkc/serialize/serializer_nil_test.go b/qkc/serialize/serializer_nil_test.go new file mode 100644 index 0000000000..6c967da3a1 --- /dev/null +++ b/qkc/serialize/serializer_nil_test.go @@ -0,0 +1,25 @@ +package serialize + +import "testing" + +// TestSerializeNilValueRejected checks that an untyped nil is rejected with an +// error rather than panicking, symmetric with DeserializeWithTags rejecting a +// nil target. A typed nil pointer is still serializable. +func TestSerializeNilValueRejected(t *testing.T) { + var w []byte + if err := Serialize(&w, nil); err == nil { + t.Fatal("expected an error serializing untyped nil") + } + if _, err := SerializeToBytes(nil); err == nil { + t.Fatal("expected an error from SerializeToBytes(nil)") + } + + // A typed nil pointer carries a type, so it still serializes (as its + // element's zero value). + type foo struct{ X uint32 } + var p *foo + w = nil + if err := Serialize(&w, p); err != nil { + t.Fatalf("typed nil pointer should serialize: %v", err) + } +} diff --git a/qkc/serialize/serializer_test.go b/qkc/serialize/serializer_test.go new file mode 100644 index 0000000000..ad95de9499 --- /dev/null +++ b/qkc/serialize/serializer_test.go @@ -0,0 +1,213 @@ +// Ported verbatim from github.com/QuarkChain/goquarkchain/serialize (byte-compatible). + +package serialize + +import ( + "bytes" + "errors" + "fmt" + "math/big" + "testing" +) + +type serializableStruct struct { + val uint64 + err error +} + +func (e *serializableStruct) Serialize(w *[]byte) error { + if e.err != nil { + return e.err + } else { + *w = append(*w, new(big.Int).SetUint64(e.val).Bytes()...) + } + return nil +} + +func (e *serializableStruct) Deserialize(bb *ByteBuffer) error { + if e == nil { + + } else if bb.Remaining() == 4 { + e = nil + } else { + e.val = 1 + } + + return nil +} + +type structForTest struct { + From *[]byte + To *[]byte `json:"to" ser:"nil"` + IgnoredField int `json:"ignore" ser:"-"` + privateField int //private field will be Ignored +} + +func newStructForTest(from, to *[]byte) structForTest { + s := structForTest{from, to, 0, 0} + return s +} + +type testDataForSerialize struct { + val interface{} + output, error string +} + +func newUint128(val uint64) *Uint128 { + ui := new(Uint128) + ui.Value = new(big.Int).SetUint64(val) + return ui +} + +func newUint256(val uint64) *Uint256 { + ui := new(Uint256) + ui.Value = new(big.Int).SetUint64(val) + return ui +} + +type LargeBytes struct { + Value []byte `bytesizeofslicelen:"4"` +} + +var serdata = []testDataForSerialize{ + // booleans + {val: true, output: "01"}, + {val: false, output: "00"}, + + // integers + {val: uint8(0), output: "00"}, + {val: uint8(128), output: "80"}, + {val: uint16(0), output: "0000"}, + {val: uint16(128), output: "0080"}, + {val: uint32(0), output: "00000000"}, + {val: uint32(128), output: "00000080"}, + {val: uint32(1024), output: "00000400"}, + {val: uint32(0xFFFFFFFF), output: "FFFFFFFF"}, + {val: uint64(0), output: "0000000000000000"}, + {val: uint64(128), output: "0000000000000080"}, + {val: uint64(0xFFFFFFFF), output: "00000000FFFFFFFF"}, + {val: uint64(0xFFFFFFFFFFFFFFFF), output: "FFFFFFFFFFFFFFFF"}, + {val: uint(0), output: "00"}, + {val: uint(128), output: "0180"}, + {val: uint(1024), output: "020400"}, + {val: uint(0xFFFFFFFF), output: "04FFFFFFFF"}, + + {val: newUint128(0), output: "00000000000000000000000000000000"}, + {val: newUint128(128), output: "00000000000000000000000000000080"}, + {val: newUint128(0xFFFFFFFFFFFFFFFF), output: "0000000000000000FFFFFFFFFFFFFFFF"}, + {val: newUint256(0), output: "0000000000000000000000000000000000000000000000000000000000000000"}, + {val: newUint256(128), output: "0000000000000000000000000000000000000000000000000000000000000080"}, + {val: newUint256(0xFFFFFFFFFFFFFFFF), output: "000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFF"}, + + // big integers (should match uint for small values) + {val: big.NewInt(0), output: "00"}, + {val: big.NewInt(1), output: "0101"}, + {val: big.NewInt(128), output: "0180"}, + {val: big.NewInt(256), output: "020100"}, + {val: big.NewInt(1024), output: "020400"}, + {val: big.NewInt(0xFFFFFF), output: "03FFFFFF"}, + {val: big.NewInt(0xFFFFFFFFFFFFFF), output: "07FFFFFFFFFFFFFF"}, + { + val: big.NewInt(0).SetBytes(unhex("102030405060708090A0B0C0D0E0F2")), + output: "0F102030405060708090A0B0C0D0E0F2", + }, + { + val: big.NewInt(0).SetBytes(unhex("0100020003000400050006000700080009000A000B000C000D000E01")), + output: "1C0100020003000400050006000700080009000A000B000C000D000E01", + }, + + // non-pointer big.Int + {val: *big.NewInt(0), output: "00"}, + {val: *big.NewInt(0xFFFFFF), output: "03FFFFFF"}, + + // negative ints are not supported + {val: big.NewInt(-1), error: "ser: cannot serialize negative *big.Int"}, + + // byte slices, strings + {val: []byte{}, output: "00"}, + {val: []byte{0x7E}, output: "017E"}, + {val: []byte{0x80}, output: "0180"}, + {val: []byte{1, 2, 3}, output: "03010203"}, + + {val: &LargeBytes{}, output: "00000000"}, + {val: &LargeBytes{[]byte{0x7E}}, output: "000000017E"}, + {val: &LargeBytes{[]byte{0x80}}, output: "0000000180"}, + {val: &LargeBytes{[]byte{1, 2, 3}}, output: "00000003010203"}, + + {val: "", output: "00000000"}, + {val: "\x7E", output: "000000017E"}, + {val: "\x80", output: "0000000180"}, + {val: "dog", output: "00000003646F67"}, + + // slices + {val: []uint8{}, output: "00"}, + {val: []uint8{1, 2, 3}, output: "03010203"}, + {val: []uint32{}, output: "00"}, + {val: []uint32{1, 2, 3}, output: "03000000010000000200000003"}, + + //Array + {val: [3]uint8{1, 2, 3}, output: "010203"}, + {val: [3]uint32{1, 2, 3}, output: "000000010000000200000003"}, + + // structs + {val: newStructForTest(&[]byte{1, 2, 3}, nil), output: "0301020300"}, + {val: newStructForTest(&[]byte{1, 2, 3}, &[]byte{4, 5, 6}), output: "030102030103040506"}, + {val: newStructForTest(nil, &[]byte{4, 5, 6}), output: "000103040506"}, + + // nil + // as nilOk default value is false, serialize will use default + // value to serialize instead of nil + {val: (*uint)(nil), output: "00"}, + {val: (*string)(nil), output: "00000000"}, + {val: (*[]byte)(nil), output: "00"}, + {val: (*[10]byte)(nil), output: "00000000000000000000"}, + {val: (*big.Int)(nil), output: "00"}, + {val: (*[]string)(nil), output: "00"}, + {val: (*[10]string)(nil), output: "00000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + // Exported field so the element encodes to >= 1 byte: a slice whose element + // encodes to zero bytes (e.g. the embedded-unexported struct{ uint }) is now + // rejected at registration — see TestZeroByteListElementRejected. + {val: (*[]struct{ X uint })(nil), output: "00"}, + + // interfaces + // Serializer + {val: (*serializableStruct)(nil), output: ""}, + {val: &serializableStruct{val: 0xFFFF}, output: "FFFF"}, + {val: &serializableStruct{1, errors.New("test error")}, error: "test error"}, + + // int is not support + {val: int(0), error: "type int is not serializable"}, + {val: (*interface{})(nil), error: "type interface {} is not serializable"}, +} + +func runEncTests(t *testing.T, f func(val interface{}) ([]byte, error)) { + for i, test := range serdata { + output, err := f(test.val) + if err != nil && test.error == "" { + t.Errorf("test %d: unexpected error: %v\nvalue %#v\ntype %T", + i, err, test.val, test.val) + continue + } + if test.error != "" && fmt.Sprint(err) != test.error { + t.Errorf("test %d: error mismatch\ngot %v\nwant %v\nvalue %#v\ntype %T", + i, err, test.error, test.val, test.val) + continue + } + if err == nil && !bytes.Equal(output, unhex(test.output)) { + t.Errorf("test %d: output mismatch:\ngot %X\nwant %s\nvalue %#v\ntype %T", + i, output, test.output, test.val, test.val) + } + } +} + +func TestSerialize(t *testing.T) { + runEncTests(t, func(val interface{}) ([]byte, error) { + b := make([]byte, 0, 1) + err := Serialize(&b, val) + return b, err + }) +} + +func TestSerializeToBytes(t *testing.T) { + runEncTests(t, SerializeToBytes) +} diff --git a/qkc/serialize/typecache.go b/qkc/serialize/typecache.go new file mode 100644 index 0000000000..f2a0d09f74 --- /dev/null +++ b/qkc/serialize/typecache.go @@ -0,0 +1,224 @@ +// Ported verbatim from github.com/QuarkChain/goquarkchain/serialize (byte-compatible). +// Modified from go-ethereum under GNU Lesser General Public License + +package serialize + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "sync" +) + +var ( + typeCacheMutex sync.RWMutex + typeCache = make(map[reflect.Type]*typeinfo) + fieldsCacheMutex sync.RWMutex + fieldsCache = make(map[reflect.Type][]field) +) + +type typeinfo struct { + serializer + deserializer +} + +// represents struct Tags +type Tags struct { + // ser:"nil" controls whether empty input results in a nil pointer. + NilOK bool + // ser:"-" ignores fields. + Ignored bool + // bytesizeofslicelen: number + ByteSizeOfSliceLen int +} + +type deserializer func(*ByteBuffer, reflect.Value, Tags) error + +type serializer func(reflect.Value, *[]byte, Tags) error + +func cachedTypeInfo(typ reflect.Type) (*typeinfo, error) { + typeCacheMutex.RLock() + info := typeCache[typ] + typeCacheMutex.RUnlock() + if info != nil { + return info, nil + } + + typeCacheMutex.Lock() + defer typeCacheMutex.Unlock() + + info, err := genTypeInfo(typ) + if err != nil { + return nil, err + } + + typeCache[typ] = info + return typeCache[typ], err +} + +type field struct { + index int + info *typeinfo + tags Tags + name string +} + +func structFields(typ reflect.Type) (fields []field, err error) { + fieldsCacheMutex.RLock() + flds := fieldsCache[typ] + fieldsCacheMutex.RUnlock() + if flds != nil { + return flds, nil + } + for i := 0; i < typ.NumField(); i++ { + if f := typ.Field(i); f.PkgPath == "" { // exported + tags, err := parseStructTag(typ, i) + if err != nil { + return nil, err + } + if tags.Ignored { + continue + } + info, err := cachedTypeInfo(f.Type) + if err != nil { + return nil, err + } + fields = append(fields, field{i, info, tags, f.Name}) + } + } + fieldsCacheMutex.Lock() + defer fieldsCacheMutex.Unlock() + fieldsCache[typ] = fields + + return fields, nil +} + +func parseStructTag(typ reflect.Type, fi int) (Tags, error) { + f := typ.Field(fi) + var ts Tags + ts.ByteSizeOfSliceLen = 1 + for _, t := range strings.Split(f.Tag.Get("ser"), ",") { + switch t = strings.TrimSpace(t); t { + case "": + case "-": + ts.Ignored = true + case "nil": // nil equal to optional in PyQuackChain + ts.NilOK = true + default: + return ts, fmt.Errorf("ser: unknown struct tag %q on %v.%s", t, typ, f.Name) + } + } + // bytesizeofslicelen use to specify the number of bytes used to save a slice len + // only slice is useful + if f.Type.Kind() == reflect.Slice { + for _, t := range strings.Split(f.Tag.Get("bytesizeofslicelen"), ",") { + t = strings.TrimSpace(t) + if t != "" { + num, err := strconv.Atoi(t) + if err != nil { + return ts, err + } + + ts.ByteSizeOfSliceLen = num + } + } + } + + return ts, nil +} + +// encodesZeroBytes reports whether a value of typ, serialized as a list element +// (default tags, i.e. no nil marker), can encode to zero bytes — e.g. struct{}, +// a struct whose fields are all ignored/unexported, a zero-length array, or a +// pointer/array/struct recursively composed of those. genTypeInfo rejects slices +// of such elements: they carry no per-element byte cost, which both makes the +// bytes-remaining bound in deserializeList reject a valid round-trip and leaves +// that bound unable to limit allocation. +// +// It mirrors makeSerializer's dispatch and must stay lock-free (no cachedTypeInfo +// / structFields), as it runs inside genTypeInfo while typeCacheMutex is held. +// The visited set breaks reference cycles, conservatively treating an in-progress +// type as non-zero. Serializable types are assumed non-zero, since their custom +// encoding can't be inspected statically (QKC's all write >= 1 byte). +func encodesZeroBytes(typ reflect.Type, visited map[reflect.Type]bool) bool { + if visited[typ] { + return false + } + // Mark typ only for the current recursion path and unmark on return, so the + // set tracks ancestors, not every type ever seen. Without the delete, the + // same zero-byte type used as repeated sibling fields (e.g. + // struct{ A empty; B empty }) would hit a stale mark on its second occurrence + // and be misclassified as non-zero, breaking serialize/deserialize symmetry. + // A type still on the path is a genuine cycle, kept conservatively as non-zero. + visited[typ] = true + defer delete(visited, typ) + + switch { + case typ.Kind() == reflect.Ptr: + // A list element pointer carries no nil marker, so it costs exactly what + // its element costs. + return encodesZeroBytes(typ.Elem(), visited) + case reflect.PtrTo(typ).Implements(serializableInterface): + return false + case typ.AssignableTo(bigInt): + return false + case isUint(typ.Kind()): + return false + case typ.Kind() == reflect.Bool: + return false + case typ.Kind() == reflect.String: + return false + case typ.Kind() == reflect.Slice: + // A slice always writes a length prefix (>= 1 byte). + return false + case typ.Kind() == reflect.Array: + if typ.Len() == 0 { + return true + } + if isByte(typ.Elem()) { + return false + } + return encodesZeroBytes(typ.Elem(), visited) + case typ.Kind() == reflect.Struct: + for i := 0; i < typ.NumField(); i++ { + f := typ.Field(i) + if f.PkgPath != "" { // unexported fields are skipped by the codec + continue + } + tags, err := parseStructTag(typ, i) + if err != nil || tags.Ignored { + continue + } + if tags.NilOK { + return false // a nil marker byte is always written + } + if !encodesZeroBytes(f.Type, visited) { + return false + } + } + return true + default: + return false + } +} + +func genTypeInfo(typ reflect.Type) (info *typeinfo, err error) { + info = new(typeinfo) + if info.serializer, err = makeSerializer(typ); err != nil { + return nil, err + } + if info.deserializer, err = makeDeserializer(typ); err != nil { + return nil, err + } + // A slice whose element encodes to zero bytes is unsupported: such a list + // serializes to just its length prefix, so deserializeList's bytes-remaining + // bound would reject the valid round-trip (and could not bound allocation + // anyway). Reject it symmetrically here so serialize and deserialize agree. + // Byte slices are exempt (a byte is one byte); arrays are exempt (their length + // is fixed by the type, not read from input, so there is nothing to bound). + if typ.Kind() == reflect.Slice && !isByte(typ.Elem()) && encodesZeroBytes(typ.Elem(), make(map[reflect.Type]bool)) { + return nil, fmt.Errorf("ser: list element type %v encodes to zero bytes, which is unsupported", typ.Elem()) + } + return info, nil +} diff --git a/qkc/serialize/typecache_zerobyte_test.go b/qkc/serialize/typecache_zerobyte_test.go new file mode 100644 index 0000000000..c4bdaa4b14 --- /dev/null +++ b/qkc/serialize/typecache_zerobyte_test.go @@ -0,0 +1,86 @@ +package serialize + +import "testing" + +// allIgnoredFields encodes to zero bytes: A is ignored, b is unexported. +type allIgnoredFields struct { + A int `ser:"-"` + b int +} + +// TestZeroByteListElementRejected pins the contract that a slice whose element +// encodes to zero bytes is unsupported and is rejected SYMMETRICALLY — both +// serialize and deserialize must fail. (Previously serialize emitted just the +// length prefix while deserialize refused it, an encode/decode asymmetry.) +func TestZeroByteListElementRejected(t *testing.T) { + cases := []struct { + name string + ser func() error + des func() error + }{ + { + "struct{}", + func() error { _, e := SerializeToBytes([]struct{}{{}, {}}); return e }, + func() error { var v []struct{}; return DeserializeFromBytes([]byte{0x02}, &v) }, + }, + { + "all ignored/unexported fields", + func() error { _, e := SerializeToBytes([]allIgnoredFields{{}, {}}); return e }, + func() error { var v []allIgnoredFields; return DeserializeFromBytes([]byte{0x02}, &v) }, + }, + { + "zero-length array element", + func() error { _, e := SerializeToBytes([][0]uint32{{}, {}}); return e }, + func() error { var v [][0]uint32; return DeserializeFromBytes([]byte{0x02}, &v) }, + }, + } + for _, c := range cases { + if err := c.ser(); err == nil { + t.Errorf("%s: SerializeToBytes should be rejected, got nil", c.name) + } + if err := c.des(); err == nil { + t.Errorf("%s: DeserializeFromBytes should be rejected, got nil", c.name) + } + } + _ = allIgnoredFields{}.b // silence unused-field linters +} + +type emptyElem struct{} + +// repeatedEmpties has two fields of the SAME zero-byte type, so it still encodes +// to zero bytes. It exercises the visited-map reset in encodesZeroBytes: a stale +// mark would misclassify the second field as non-zero, letting []repeatedEmpties +// serialize to a bare length prefix that deserialize then rejects (an asymmetry). +type repeatedEmpties struct { + A emptyElem + B emptyElem +} + +// TestZeroByteRepeatedFieldTypeRejected pins that a slice whose element repeats a +// zero-byte type is rejected SYMMETRICALLY (both serialize and deserialize fail). +func TestZeroByteRepeatedFieldTypeRejected(t *testing.T) { + if _, err := SerializeToBytes([]repeatedEmpties{{}, {}}); err == nil { + t.Error("SerializeToBytes([]repeatedEmpties) should be rejected, got nil") + } + var v []repeatedEmpties + if err := DeserializeFromBytes([]byte{0x02}, &v); err == nil { + t.Error("DeserializeFromBytes into []repeatedEmpties should be rejected, got nil") + } +} + +// TestNonZeroByteListStillRoundTrips guards against over-rejection: a slice +// whose element consumes >= 1 byte must still encode and decode normally. +func TestNonZeroByteListStillRoundTrips(t *testing.T) { + in := []uint32{7, 9} + b, err := SerializeToBytes(in) + if err != nil { + t.Fatalf("serialize: %v", err) + } + var out []uint32 + if err := DeserializeFromBytes(b, &out); err != nil { + t.Fatalf("deserialize: %v", err) + } + if len(out) != 2 || out[0] != 7 || out[1] != 9 { + t.Fatalf("round-trip mismatch: %v", out) + } +} diff --git a/qkc/serialize/utils.go b/qkc/serialize/utils.go new file mode 100644 index 0000000000..a79ff824ec --- /dev/null +++ b/qkc/serialize/utils.go @@ -0,0 +1,58 @@ +// Ported verbatim from github.com/QuarkChain/goquarkchain/serialize (byte-compatible). + +package serialize + +import ( + "math/big" + "reflect" +) + +var ( + serializableInterface = reflect.TypeOf(new(Serializable)).Elem() + bigInt = reflect.TypeOf(big.Int{}) + typUint128 = reflect.TypeOf(Uint128{}) + typUint256 = reflect.TypeOf(Uint256{}) + big0 = big.NewInt(0) +) + +type BigUint struct { + Value *big.Int +} + +type Uint128 BigUint +type Uint256 BigUint + +func (ui *Uint128) Serialize(w *[]byte) error { + return serializeFixSizeBigUint(ui.Value, 16, w) +} + +func (ui *Uint128) Deserialize(bb *ByteBuffer) error { + if ui.Value == nil { + ui.Value = new(big.Int) + } + return deserializeFixSizeBigUint(bb, ui.Value, 16) +} + +func (ui *Uint256) Serialize(w *[]byte) error { + return serializeFixSizeBigUint(ui.Value, 32, w) +} + +func (ui *Uint256) Deserialize(bb *ByteBuffer) error { + if ui.Value == nil { + ui.Value = new(big.Int) + } + return deserializeFixSizeBigUint(bb, ui.Value, 32) +} + +type Serializable interface { + Serialize(w *[]byte) error + Deserialize(bb *ByteBuffer) error +} + +func isUint(k reflect.Kind) bool { + return k >= reflect.Uint && k <= reflect.Uintptr +} + +func isByte(typ reflect.Type) bool { + return typ.Kind() == reflect.Uint8 && !typ.Implements(serializableInterface) +} diff --git a/qkc/serialize/utils_negative_test.go b/qkc/serialize/utils_negative_test.go new file mode 100644 index 0000000000..82dd72da53 --- /dev/null +++ b/qkc/serialize/utils_negative_test.go @@ -0,0 +1,34 @@ +package serialize + +import ( + "math/big" + "testing" +) + +// TestFixedSizeUintRejectsNegative guards against silent sign loss: because +// big.Int.Bytes() returns the absolute value, a negative Uint128/Uint256 would +// otherwise serialize identically to its positive counterpart. It must error +// instead. +func TestFixedSizeUintRejectsNegative(t *testing.T) { + cases := []struct { + name string + ser func() error + }{ + {"Uint128", func() error { var w []byte; return (&Uint128{Value: big.NewInt(-1)}).Serialize(&w) }}, + {"Uint256", func() error { var w []byte; return (&Uint256{Value: big.NewInt(-1)}).Serialize(&w) }}, + } + for _, tc := range cases { + if err := tc.ser(); err == nil { + t.Fatalf("%s: expected an error serializing a negative value, got nil (sign silently dropped)", tc.name) + } + } + + // Non-negative values still serialize to their fixed width. + var w []byte + if err := (&Uint256{Value: big.NewInt(1)}).Serialize(&w); err != nil { + t.Fatalf("positive Uint256 rejected: %v", err) + } + if len(w) != 32 { + t.Fatalf("Uint256 should serialize to 32 bytes, got %d", len(w)) + } +}