mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-20 10:52:25 +00:00
trie: try out ristretto cache
This commit is contained in:
Martin Holst Swende
parent
df89233b57
commit
fa166501ad
18 changed files with 2106 additions and 30 deletions
|
|
@ -17,7 +17,6 @@
|
|||
package trie
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -25,7 +24,6 @@ import (
|
|||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/allegro/bigcache"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
|
|
@ -69,7 +67,7 @@ const secureKeyLength = 11 + 32
|
|||
type Database struct {
|
||||
diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes
|
||||
|
||||
cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs
|
||||
cleans *RistrettoCache // GC friendly memory cache of clean node RLPs
|
||||
dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes
|
||||
oldest common.Hash // Oldest tracked node, flush-list head
|
||||
newest common.Hash // Newest tracked node, flush-list tail
|
||||
|
|
@ -275,19 +273,6 @@ func expandNode(hash hashNode, n node) node {
|
|||
}
|
||||
}
|
||||
|
||||
// trienodeHasher is a struct to be used with BigCache, which uses a Hasher to
|
||||
// determine which shard to place an entry into. It's not a cryptographic hash,
|
||||
// just to provide a bit of anti-collision (default is FNV64a).
|
||||
//
|
||||
// Since trie keys are already hashes, we can just use the key directly to
|
||||
// map shard id.
|
||||
type trienodeHasher struct{}
|
||||
|
||||
// Sum64 implements the bigcache.Hasher interface.
|
||||
func (t trienodeHasher) Sum64(key string) uint64 {
|
||||
return binary.BigEndian.Uint64([]byte(key))
|
||||
}
|
||||
|
||||
// NewDatabase creates a new trie database to store ephemeral trie content before
|
||||
// its written out to disk or garbage collected. No read cache is created, so all
|
||||
// data retrievals will hit the underlying disk database.
|
||||
|
|
@ -299,16 +284,9 @@ func NewDatabase(diskdb ethdb.KeyValueStore) *Database {
|
|||
// before its written out to disk or garbage collected. It also acts as a read cache
|
||||
// for nodes loaded from disk.
|
||||
func NewDatabaseWithCache(diskdb ethdb.KeyValueStore, cache int) *Database {
|
||||
var cleans *bigcache.BigCache
|
||||
var cleans *RistrettoCache
|
||||
if cache > 0 {
|
||||
cleans, _ = bigcache.NewBigCache(bigcache.Config{
|
||||
Shards: 1024,
|
||||
LifeWindow: time.Hour,
|
||||
MaxEntriesInWindow: cache * 1024,
|
||||
MaxEntrySize: 512,
|
||||
HardMaxCacheSize: cache,
|
||||
Hasher: trienodeHasher{},
|
||||
})
|
||||
cleans, _ = NewRistrettoCache(cache)
|
||||
}
|
||||
return &Database{
|
||||
diskdb: diskdb,
|
||||
|
|
@ -384,7 +362,7 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) {
|
|||
func (db *Database) node(hash common.Hash) node {
|
||||
// Retrieve the node from the clean cache if available
|
||||
if db.cleans != nil {
|
||||
if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil {
|
||||
if enc, err := db.cleans.Get(hash); err == nil && enc != nil {
|
||||
memcacheCleanHitMeter.Mark(1)
|
||||
memcacheCleanReadMeter.Mark(int64(len(enc)))
|
||||
return mustDecodeNode(hash[:], enc)
|
||||
|
|
@ -404,7 +382,7 @@ func (db *Database) node(hash common.Hash) node {
|
|||
return nil
|
||||
}
|
||||
if db.cleans != nil {
|
||||
db.cleans.Set(string(hash[:]), enc)
|
||||
db.cleans.Set(hash, enc)
|
||||
memcacheCleanMissMeter.Mark(1)
|
||||
memcacheCleanWriteMeter.Mark(int64(len(enc)))
|
||||
}
|
||||
|
|
@ -420,7 +398,7 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
|
|||
}
|
||||
// Retrieve the node from the clean cache if available
|
||||
if db.cleans != nil {
|
||||
if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil {
|
||||
if enc, err := db.cleans.Get(hash); err == nil && enc != nil {
|
||||
memcacheCleanHitMeter.Mark(1)
|
||||
memcacheCleanReadMeter.Mark(int64(len(enc)))
|
||||
return enc, nil
|
||||
|
|
@ -438,7 +416,7 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
|
|||
enc, err := db.diskdb.Get(hash[:])
|
||||
if err == nil && enc != nil {
|
||||
if db.cleans != nil {
|
||||
db.cleans.Set(string(hash[:]), enc)
|
||||
db.cleans.Set(hash, enc)
|
||||
memcacheCleanMissMeter.Mark(1)
|
||||
memcacheCleanWriteMeter.Mark(int64(len(enc)))
|
||||
}
|
||||
|
|
@ -835,7 +813,7 @@ func (c *cleaner) Put(key []byte, rlp []byte) error {
|
|||
}
|
||||
// Move the flushed node into the clean cache to prevent insta-reloads
|
||||
if c.db.cleans != nil {
|
||||
c.db.cleans.Set(string(hash[:]), rlp)
|
||||
c.db.cleans.Set(hash, rlp)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
64
trie/ristretto.go
Normal file
64
trie/ristretto.go
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
package trie
|
||||
|
||||
// Copyright 2019 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/dgraph-io/ristretto"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
)
|
||||
|
||||
var ErrMissingItem = errors.New("missing item")
|
||||
|
||||
type RistrettoCache struct {
|
||||
cache *ristretto.Cache
|
||||
}
|
||||
|
||||
func NewRistrettoCache(capacity int) (*RistrettoCache, error) {
|
||||
cache, err := ristretto.NewCache(&ristretto.Config{
|
||||
NumCounters: int64(capacity * 10),
|
||||
MaxCost: int64(capacity),
|
||||
BufferItems: 64,
|
||||
Metrics: false,
|
||||
KeyToHash: func(key interface{}) uint64 {
|
||||
h := key.(common.Hash)
|
||||
return uint64(h[7]) | uint64(h[6])<<8 | uint64(h[5])<<16 | uint64(h[4])<<24 |
|
||||
uint64(h[3])<<32 | uint64(h[2])<<40 | uint64(h[1])<<48 | uint64(h[0])<<56
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &RistrettoCache{
|
||||
cache: cache,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
func (c *RistrettoCache) Get(key common.Hash) ([]byte, error) {
|
||||
v, exist := c.cache.Get(key)
|
||||
if exist {
|
||||
return []byte(v.(string)), nil
|
||||
}
|
||||
return nil, ErrMissingItem
|
||||
}
|
||||
|
||||
func (c *RistrettoCache) Set(key common.Hash, value []byte) error {
|
||||
c.cache.Set(key, string(value), int64(len(value)))
|
||||
return nil
|
||||
}
|
||||
176
vendor/github.com/dgraph-io/ristretto/LICENSE
generated
vendored
Normal file
176
vendor/github.com/dgraph-io/ristretto/LICENSE
generated
vendored
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
104
vendor/github.com/dgraph-io/ristretto/README.md
generated
vendored
Normal file
104
vendor/github.com/dgraph-io/ristretto/README.md
generated
vendored
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
# Ristretto
|
||||
|
||||
[](https://godoc.org/github.com/dgraph-io/ristretto)
|
||||
[](https://goreportcard.com/report/github.com/dgraph-io/ristretto)
|
||||
|
||||
Ristretto is a fast, concurrent cache library using a [TinyLFU](https://arxiv.org/abs/1512.00727)
|
||||
admission policy and Sampled LFU eviction policy.
|
||||
|
||||
The motivation to build Ristretto comes from the need for a contention-free
|
||||
cache in [Dgraph][].
|
||||
|
||||
[Dgraph]: https://github.com/dgraph-io/dgraph
|
||||
|
||||
## Example
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/dgraph-io/ristretto"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// create a cache instance
|
||||
cache, err := ristretto.NewCache(&ristretto.Config{
|
||||
NumCounters: 1000000 * 10,
|
||||
MaxCost: 1000000,
|
||||
BufferItems: 64,
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// set a value
|
||||
cache.Set("key", "value", 1)
|
||||
|
||||
// wait for value to pass through buffers
|
||||
time.Sleep(time.Second / 100)
|
||||
|
||||
// get a value, given a key
|
||||
value, found := cache.Get("key")
|
||||
if !found {
|
||||
panic("missing value")
|
||||
}
|
||||
|
||||
fmt.Println(value)
|
||||
|
||||
// delete a value, given a key
|
||||
cache.Del("key")
|
||||
}
|
||||
```
|
||||
|
||||
### Benchmarks
|
||||
|
||||
The benchmarks can be found in https://github.com/dgraph-io/benchmarks/tree/master/cachebench/ristretto
|
||||
|
||||
### Hit Ratios
|
||||
|
||||
#### Search
|
||||
|
||||
This trace is described as "disk read accesses initiated by a large commercial
|
||||
search engine in response to various web search requests."
|
||||
|
||||
.svg?sanitize=true)
|
||||
|
||||
#### Database
|
||||
|
||||
This trace is described as "a database server running at a commercial site
|
||||
running an ERP application on top of a commercial database."
|
||||
|
||||
.svg?sanitize=true)
|
||||
|
||||
#### Looping
|
||||
|
||||
This trace demonstrates a looping access pattern.
|
||||
|
||||
.svg?sanitize=true)
|
||||
|
||||
#### CODASYL
|
||||
|
||||
This trace is described as "references to a CODASYL database for a one hour
|
||||
period."
|
||||
|
||||
.svg?sanitize=true)
|
||||
|
||||
### Throughput
|
||||
|
||||
All throughput benchmarks were ran on an Intel Core i7-8700K (3.7GHz) with 16gb
|
||||
of RAM.
|
||||
|
||||
#### Mixed
|
||||
|
||||
|
||||
|
||||
#### Read
|
||||
|
||||
.svg?sanitize=true)
|
||||
|
||||
#### Write
|
||||
|
||||
.svg?sanitize=true)
|
||||
358
vendor/github.com/dgraph-io/ristretto/cache.go
generated
vendored
Normal file
358
vendor/github.com/dgraph-io/ristretto/cache.go
generated
vendored
Normal file
|
|
@ -0,0 +1,358 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Ristretto is a fast, fixed size, in-memory cache with a dual focus on
|
||||
// throughput and hit ratio performance. You can easily add Ristretto to an
|
||||
// existing system and keep the most valuable data where you need it.
|
||||
package ristretto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/dgraph-io/ristretto/z"
|
||||
)
|
||||
|
||||
// Cache is a thread-safe implementation of a hashmap with a TinyLFU admission
|
||||
// policy and a Sampled LFU eviction policy. You can use the same Cache instance
|
||||
// from as many goroutines as you want.
|
||||
type Cache struct {
|
||||
// store is the central concurrent hashmap where key-value items are stored
|
||||
store store
|
||||
// policy determines what gets let in to the cache and what gets kicked out
|
||||
policy policy
|
||||
// getBuf is a custom ring buffer implementation that gets pushed to when
|
||||
// keys are read
|
||||
getBuf *ringBuffer
|
||||
// setBuf is a buffer allowing us to batch/drop Sets during times of high
|
||||
// contention
|
||||
setBuf chan *item
|
||||
// stats contains a running log of important statistics like hits, misses,
|
||||
// and dropped items
|
||||
stats *metrics
|
||||
// onEvict is called for item evictions
|
||||
onEvict func(uint64, interface{}, int64)
|
||||
// KeyToHash function is used to customize the key hashing algorithm.
|
||||
// Each key will be hashed using the provided function. If keyToHash value
|
||||
// is not set, the default keyToHash function is used.
|
||||
keyToHash func(interface{}) uint64
|
||||
}
|
||||
|
||||
// Config is passed to NewCache for creating new Cache instances.
|
||||
type Config struct {
|
||||
// NumCounters determines the number of counters (keys) to keep that hold
|
||||
// access frequency information. It's generally a good idea to have more
|
||||
// counters than the max cache capacity, as this will improve eviction
|
||||
// accuracy and subsequent hit ratios.
|
||||
//
|
||||
// For example, if you expect your cache to hold 1,000,000 items when full,
|
||||
// NumCounters should be 10,000,000 (10x). Each counter takes up 4 bits, so
|
||||
// keeping 10,000,000 counters would require 5MB of memory.
|
||||
NumCounters int64
|
||||
// MaxCost can be considered as the cache capacity, in whatever units you
|
||||
// choose to use.
|
||||
//
|
||||
// For example, if you want the cache to have a max capacity of 100MB, you
|
||||
// would set MaxCost to 100,000,000 and pass an item's number of bytes as
|
||||
// the `cost` parameter for calls to Set. If new items are accepted, the
|
||||
// eviction process will take care of making room for the new item and not
|
||||
// overflowing the MaxCost value.
|
||||
MaxCost int64
|
||||
// BufferItems determines the size of Get buffers.
|
||||
//
|
||||
// Unless you have a rare use case, using `64` as the BufferItems value
|
||||
// results in good performance.
|
||||
BufferItems int64
|
||||
// Metrics determines whether cache statistics are kept during the cache's
|
||||
// lifetime. There *is* some overhead to keeping statistics, so you should
|
||||
// only set this flag to true when testing or throughput performance isn't a
|
||||
// major factor.
|
||||
Metrics bool
|
||||
// OnEvict is called for every eviction and passes the hashed key, value,
|
||||
// and cost to the function.
|
||||
OnEvict func(key uint64, value interface{}, cost int64)
|
||||
// KeyToHash function is used to customize the key hashing algorithm.
|
||||
// Each key will be hashed using the provided function. If keyToHash value
|
||||
// is not set, the default keyToHash function is used.
|
||||
KeyToHash func(key interface{}) uint64
|
||||
}
|
||||
|
||||
// item is passed to setBuf so items can eventually be added to the cache
|
||||
type item struct {
|
||||
key uint64
|
||||
val interface{}
|
||||
cost int64
|
||||
}
|
||||
|
||||
// NewCache returns a new Cache instance and any configuration errors, if any.
|
||||
func NewCache(config *Config) (*Cache, error) {
|
||||
switch {
|
||||
case config.NumCounters == 0:
|
||||
return nil, errors.New("NumCounters can't be zero.")
|
||||
case config.MaxCost == 0:
|
||||
return nil, errors.New("MaxCost can't be zero.")
|
||||
case config.BufferItems == 0:
|
||||
return nil, errors.New("BufferItems can't be zero.")
|
||||
}
|
||||
policy := newPolicy(config.NumCounters, config.MaxCost)
|
||||
cache := &Cache{
|
||||
store: newStore(),
|
||||
policy: policy,
|
||||
getBuf: newRingBuffer(ringLossy, &ringConfig{
|
||||
Consumer: policy,
|
||||
Capacity: config.BufferItems,
|
||||
}),
|
||||
// TODO: size configuration for this? like BufferItems but for setBuf?
|
||||
setBuf: make(chan *item, 32*1024),
|
||||
onEvict: config.OnEvict,
|
||||
keyToHash: config.KeyToHash,
|
||||
}
|
||||
if config.Metrics {
|
||||
cache.collectMetrics()
|
||||
}
|
||||
// We can possibly make this configurable. But having 2 goroutines
|
||||
// processing this seems sufficient for now.
|
||||
//
|
||||
// TODO: Allow a way to stop these goroutines.
|
||||
for i := 0; i < 2; i++ {
|
||||
go cache.processItems()
|
||||
}
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
// Get returns the value (if any) and a boolean representing whether the
|
||||
// value was found or not. The value can be nil and the boolean can be true at
|
||||
// the same time.
|
||||
func (c *Cache) Get(key interface{}) (interface{}, bool) {
|
||||
if c == nil {
|
||||
return nil, false
|
||||
}
|
||||
hash := c.keyHash(key)
|
||||
c.getBuf.Push(hash)
|
||||
val, ok := c.store.Get(hash)
|
||||
if ok {
|
||||
c.stats.Add(hit, hash, 1)
|
||||
} else {
|
||||
c.stats.Add(miss, hash, 1)
|
||||
}
|
||||
return val, ok
|
||||
}
|
||||
|
||||
// keyHash generates the hash for a given key using the cutom keyToHash function, if provided.
|
||||
// Otherwise it generates the hash using the z.KeyToHash funcion.
|
||||
func (c *Cache) keyHash(key interface{}) uint64 {
|
||||
if c.keyToHash != nil {
|
||||
return c.keyToHash(key)
|
||||
}
|
||||
return z.KeyToHash(key)
|
||||
}
|
||||
|
||||
// Set attempts to add the key-value item to the cache. If it returns false,
|
||||
// then the Set was dropped and the key-value item isn't added to the cache. If
|
||||
// it returns true, there's still a chance it could be dropped by the policy if
|
||||
// its determined that the key-value item isn't worth keeping, but otherwise the
|
||||
// item will be added and other items will be evicted in order to make room.
|
||||
func (c *Cache) Set(key interface{}, val interface{}, cost int64) bool {
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
hash := c.keyHash(key)
|
||||
// TODO: Add a c.store.UpdateIfPresent here. This would catch any value updates and avoid having
|
||||
// to push the key in setBuf.
|
||||
|
||||
// attempt to add the (possibly) new item to the setBuf where it will later
|
||||
// be processed by the policy and evaluated
|
||||
select {
|
||||
case c.setBuf <- &item{key: hash, val: val, cost: cost}:
|
||||
return true
|
||||
default:
|
||||
// drop the set and avoid blocking
|
||||
c.stats.Add(dropSets, hash, 1)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Add a public Update function, which would update a key only if present.
|
||||
|
||||
// Del deletes the key-value item from the cache if it exists.
|
||||
func (c *Cache) Del(key interface{}) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
hash := c.keyHash(key)
|
||||
c.policy.Del(hash)
|
||||
c.store.Del(hash)
|
||||
}
|
||||
|
||||
// Close stops all goroutines and closes all channels.
|
||||
func (c *Cache) Close() {}
|
||||
|
||||
// processItems is ran by goroutines processing the Set buffer.
|
||||
func (c *Cache) processItems() {
|
||||
for item := range c.setBuf {
|
||||
victims, added := c.policy.Add(item.key, item.cost)
|
||||
if added {
|
||||
// item was accepted by the policy, so add to the hashmap
|
||||
c.store.Set(item.key, item.val)
|
||||
}
|
||||
// delete victims that are no longer worthy of being in the cache
|
||||
for _, victim := range victims {
|
||||
// eviction callback
|
||||
if c.onEvict != nil {
|
||||
victim.val, _ = c.store.Get(victim.key)
|
||||
c.onEvict(victim.key, victim.val, victim.cost)
|
||||
}
|
||||
// delete from hashmap
|
||||
c.store.Del(victim.key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache) collectMetrics() {
|
||||
c.stats = newMetrics()
|
||||
c.policy.CollectMetrics(c.stats)
|
||||
}
|
||||
|
||||
// Metrics returns statistics about cache performance.
|
||||
func (c *Cache) Metrics() *metrics {
|
||||
return c.stats
|
||||
}
|
||||
|
||||
type metricType int
|
||||
|
||||
const (
|
||||
// The following 2 keep track of hits and misses.
|
||||
hit = iota
|
||||
miss
|
||||
|
||||
// The following 3 keep track of number of keys added, updated and evicted.
|
||||
keyAdd
|
||||
keyUpdate
|
||||
keyEvict
|
||||
|
||||
// The following 2 keep track of cost of keys added and evicted.
|
||||
costAdd
|
||||
costEvict
|
||||
|
||||
// The following keep track of how many sets were dropped or rejected later.
|
||||
dropSets
|
||||
rejectSets
|
||||
|
||||
// The following 2 keep track of how many gets were kept and dropped on the floor.
|
||||
dropGets
|
||||
keepGets
|
||||
|
||||
// This should be the final enum. Other enums should be set before this.
|
||||
doNotUse
|
||||
)
|
||||
|
||||
func stringFor(t metricType) string {
|
||||
switch t {
|
||||
case hit:
|
||||
return "hit"
|
||||
case miss:
|
||||
return "miss"
|
||||
case keyAdd:
|
||||
return "keys-added"
|
||||
case keyUpdate:
|
||||
return "keys-updated"
|
||||
case keyEvict:
|
||||
return "keys-evicted"
|
||||
case costAdd:
|
||||
return "cost-added"
|
||||
case costEvict:
|
||||
return "cost-evicted"
|
||||
case dropSets:
|
||||
return "sets-dropped"
|
||||
case rejectSets:
|
||||
return "sets-rejected" // by policy.
|
||||
case dropGets:
|
||||
return "gets-dropped"
|
||||
case keepGets:
|
||||
return "gets-kept"
|
||||
default:
|
||||
return "unidentified"
|
||||
}
|
||||
}
|
||||
|
||||
// metrics is the struct for hit ratio statistics. Note that there is some
|
||||
// cost to maintaining the counters, so it's best to wrap Policies via the
|
||||
// Recorder type when hit ratio analysis is needed.
|
||||
type metrics struct {
|
||||
all [doNotUse][]*uint64
|
||||
}
|
||||
|
||||
func newMetrics() *metrics {
|
||||
s := &metrics{}
|
||||
for i := 0; i < doNotUse; i++ {
|
||||
s.all[i] = make([]*uint64, 256)
|
||||
slice := s.all[i]
|
||||
for j := range slice {
|
||||
slice[j] = new(uint64)
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (p *metrics) Add(t metricType, hash, delta uint64) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
valp := p.all[t]
|
||||
// Avoid false sharing by padding at least 64 bytes of space between two
|
||||
// atomic counters which would be incremented.
|
||||
idx := (hash % 25) * 10
|
||||
atomic.AddUint64(valp[idx], delta)
|
||||
}
|
||||
|
||||
func (p *metrics) Get(t metricType) uint64 {
|
||||
if p == nil {
|
||||
return 0
|
||||
}
|
||||
valp := p.all[t]
|
||||
var total uint64
|
||||
for i := range valp {
|
||||
total += atomic.LoadUint64(valp[i])
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (p *metrics) Ratio() float64 {
|
||||
if p == nil {
|
||||
return 0.0
|
||||
}
|
||||
hits, misses := p.Get(hit), p.Get(miss)
|
||||
if hits == 0 && misses == 0 {
|
||||
return 0.0
|
||||
}
|
||||
return float64(hits) / float64(hits+misses)
|
||||
}
|
||||
|
||||
func (p *metrics) String() string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
for i := 0; i < doNotUse; i++ {
|
||||
t := metricType(i)
|
||||
fmt.Fprintf(&buf, "%s: %d ", stringFor(t), p.Get(t))
|
||||
}
|
||||
fmt.Fprintf(&buf, "gets-total: %d ", p.Get(hit)+p.Get(miss))
|
||||
fmt.Fprintf(&buf, "hit-ratio: %.2f", p.Ratio())
|
||||
return buf.String()
|
||||
}
|
||||
9
vendor/github.com/dgraph-io/ristretto/go.mod
generated
vendored
Normal file
9
vendor/github.com/dgraph-io/ristretto/go.mod
generated
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
module github.com/dgraph-io/ristretto
|
||||
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2
|
||||
github.com/stretchr/testify v1.3.0
|
||||
)
|
||||
10
vendor/github.com/dgraph-io/ristretto/go.sum
generated
vendored
Normal file
10
vendor/github.com/dgraph-io/ristretto/go.sum
generated
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
411
vendor/github.com/dgraph-io/ristretto/policy.go
generated
vendored
Normal file
411
vendor/github.com/dgraph-io/ristretto/policy.go
generated
vendored
Normal file
|
|
@ -0,0 +1,411 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package ristretto
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"math"
|
||||
"sync"
|
||||
|
||||
"github.com/dgraph-io/ristretto/z"
|
||||
)
|
||||
|
||||
const (
|
||||
// lfuSample is the number of items to sample when looking at eviction
|
||||
// candidates. 5 seems to be the most optimal number [citation needed].
|
||||
lfuSample = 5
|
||||
)
|
||||
|
||||
// policy is the interface encapsulating eviction/admission behavior.
|
||||
type policy interface {
|
||||
ringConsumer
|
||||
// Add attempts to Add the key-cost pair to the Policy. It returns a slice
|
||||
// of evicted keys and a bool denoting whether or not the key-cost pair
|
||||
// was added. If it returns true, the key should be stored in cache.
|
||||
Add(uint64, int64) ([]*item, bool)
|
||||
// Has returns true if the key exists in the Policy.
|
||||
Has(uint64) bool
|
||||
// Del deletes the key from the Policy.
|
||||
Del(uint64)
|
||||
// Cap returns the available capacity.
|
||||
Cap() int64
|
||||
// Optionally, set stats object to track how policy is performing.
|
||||
CollectMetrics(stats *metrics)
|
||||
}
|
||||
|
||||
func newPolicy(numCounters, maxCost int64) policy {
|
||||
p := &defaultPolicy{
|
||||
admit: newTinyLFU(numCounters),
|
||||
evict: newSampledLFU(maxCost),
|
||||
itemsCh: make(chan []uint64, 3),
|
||||
}
|
||||
// TODO: Add a way to stop the goroutine.
|
||||
go p.processItems()
|
||||
return p
|
||||
}
|
||||
|
||||
// defaultPolicy is the default defaultPolicy, which is currently TinyLFU
|
||||
// admission with sampledLFU eviction.
|
||||
type defaultPolicy struct {
|
||||
sync.Mutex
|
||||
admit *tinyLFU
|
||||
evict *sampledLFU
|
||||
itemsCh chan []uint64
|
||||
stats *metrics
|
||||
}
|
||||
|
||||
func (p *defaultPolicy) CollectMetrics(stats *metrics) {
|
||||
p.stats = stats
|
||||
p.evict.stats = stats
|
||||
}
|
||||
|
||||
type policyPair struct {
|
||||
key uint64
|
||||
cost int64
|
||||
}
|
||||
|
||||
func (p *defaultPolicy) processItems() {
|
||||
for items := range p.itemsCh {
|
||||
p.Lock()
|
||||
p.admit.Push(items)
|
||||
p.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *defaultPolicy) Push(keys []uint64) bool {
|
||||
if len(keys) == 0 {
|
||||
return true
|
||||
}
|
||||
select {
|
||||
case p.itemsCh <- keys:
|
||||
p.stats.Add(keepGets, keys[0], uint64(len(keys)))
|
||||
return true
|
||||
default:
|
||||
p.stats.Add(dropGets, keys[0], uint64(len(keys)))
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *defaultPolicy) Add(key uint64, cost int64) ([]*item, bool) {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
// can't add an item bigger than entire cache
|
||||
if cost > p.evict.maxCost {
|
||||
return nil, false
|
||||
}
|
||||
// we don't need to go any further if the item is already in the cache
|
||||
if has := p.evict.updateIfHas(key, cost); has {
|
||||
return nil, true
|
||||
}
|
||||
// if we got this far, this key doesn't exist in the cache
|
||||
//
|
||||
// calculate the remaining room in the cache (usually bytes)
|
||||
room := p.evict.roomLeft(cost)
|
||||
if room >= 0 {
|
||||
// there's enough room in the cache to store the new item without
|
||||
// overflowing, so we can do that now and stop here
|
||||
p.evict.add(key, cost)
|
||||
return nil, true
|
||||
}
|
||||
// incHits is the hit count for the incoming item
|
||||
incHits := p.admit.Estimate(key)
|
||||
// sample is the eviction candidate pool to be filled via random sampling
|
||||
//
|
||||
// TODO: perhaps we should use a min heap here. Right now our time
|
||||
// complexity is N for finding the min. Min heap should bring it down to
|
||||
// O(lg N).
|
||||
sample := make([]*policyPair, 0, lfuSample)
|
||||
// as items are evicted they will be appended to victims
|
||||
victims := make([]*item, 0)
|
||||
// Delete victims until there's enough space or a minKey is found that has
|
||||
// more hits than incoming item.
|
||||
for ; room < 0; room = p.evict.roomLeft(cost) {
|
||||
// fill up empty slots in sample
|
||||
sample = p.evict.fillSample(sample)
|
||||
// find minimally used item in sample
|
||||
minKey, minHits, minId, minCost := uint64(0), int64(math.MaxInt64), 0, int64(0)
|
||||
for i, pair := range sample {
|
||||
// look up hit count for sample key
|
||||
if hits := p.admit.Estimate(pair.key); hits < minHits {
|
||||
minKey, minHits, minId, minCost = pair.key, hits, i, pair.cost
|
||||
}
|
||||
}
|
||||
// If the incoming item isn't worth keeping in the policy, reject.
|
||||
if incHits < minHits {
|
||||
p.stats.Add(rejectSets, key, 1)
|
||||
return victims, false
|
||||
}
|
||||
// delete the victim from metadata
|
||||
p.evict.del(minKey)
|
||||
// delete the victim from sample
|
||||
sample[minId] = sample[len(sample)-1]
|
||||
sample = sample[:len(sample)-1]
|
||||
// store victim in evicted victims slice
|
||||
victims = append(victims, &item{minKey, nil, minCost})
|
||||
}
|
||||
p.evict.add(key, cost)
|
||||
return victims, true
|
||||
}
|
||||
|
||||
func (p *defaultPolicy) Has(key uint64) bool {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
_, exists := p.evict.keyCosts[key]
|
||||
return exists
|
||||
}
|
||||
|
||||
func (p *defaultPolicy) Del(key uint64) {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
p.evict.del(key)
|
||||
}
|
||||
|
||||
func (p *defaultPolicy) Cap() int64 {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
return int64(p.evict.maxCost - p.evict.used)
|
||||
}
|
||||
|
||||
// sampledLFU is an eviction helper storing key-cost pairs.
|
||||
type sampledLFU struct {
|
||||
keyCosts map[uint64]int64
|
||||
maxCost int64
|
||||
used int64
|
||||
stats *metrics
|
||||
}
|
||||
|
||||
func newSampledLFU(maxCost int64) *sampledLFU {
|
||||
return &sampledLFU{
|
||||
keyCosts: make(map[uint64]int64),
|
||||
maxCost: maxCost,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *sampledLFU) roomLeft(cost int64) int64 {
|
||||
return p.maxCost - (p.used + cost)
|
||||
}
|
||||
|
||||
func (p *sampledLFU) fillSample(in []*policyPair) []*policyPair {
|
||||
if len(in) >= lfuSample {
|
||||
return in
|
||||
}
|
||||
for key, cost := range p.keyCosts {
|
||||
in = append(in, &policyPair{key, cost})
|
||||
if len(in) >= lfuSample {
|
||||
return in
|
||||
}
|
||||
}
|
||||
return in
|
||||
}
|
||||
|
||||
func (p *sampledLFU) del(key uint64) {
|
||||
cost, ok := p.keyCosts[key]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
p.stats.Add(keyEvict, key, 1)
|
||||
p.stats.Add(costEvict, key, uint64(cost))
|
||||
|
||||
p.used -= cost
|
||||
delete(p.keyCosts, key)
|
||||
}
|
||||
|
||||
func (p *sampledLFU) add(key uint64, cost int64) {
|
||||
p.stats.Add(keyAdd, key, 1)
|
||||
p.stats.Add(costAdd, key, uint64(cost))
|
||||
|
||||
p.keyCosts[key] = cost
|
||||
p.used += cost
|
||||
}
|
||||
|
||||
// TODO: Move this to the store itself. So, it can be used by public Set.
|
||||
func (p *sampledLFU) updateIfHas(key uint64, cost int64) (updated bool) {
|
||||
if prev, exists := p.keyCosts[key]; exists {
|
||||
// Update the cost of the existing key. For simplicity, don't worry about evicting anything
|
||||
// if the updated cost causes the size to grow beyond maxCost.
|
||||
p.stats.Add(keyUpdate, key, 1)
|
||||
p.used += cost - prev
|
||||
p.keyCosts[key] = cost
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// tinyLFU is an admission helper that keeps track of access frequency using
|
||||
// tiny (4-bit) counters in the form of a count-min sketch.
|
||||
// tinyLFU is NOT thread safe.
|
||||
type tinyLFU struct {
|
||||
freq *cmSketch
|
||||
door *z.Bloom
|
||||
incrs int64
|
||||
resetAt int64
|
||||
}
|
||||
|
||||
func newTinyLFU(numCounters int64) *tinyLFU {
|
||||
return &tinyLFU{
|
||||
freq: newCmSketch(numCounters),
|
||||
door: z.NewBloomFilter(float64(numCounters), 0.01),
|
||||
resetAt: numCounters,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *tinyLFU) Push(keys []uint64) {
|
||||
for _, key := range keys {
|
||||
p.Increment(key)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *tinyLFU) Estimate(key uint64) int64 {
|
||||
hits := p.freq.Estimate(key)
|
||||
if p.door.Has(key) {
|
||||
hits += 1
|
||||
}
|
||||
return hits
|
||||
}
|
||||
|
||||
func (p *tinyLFU) Increment(key uint64) {
|
||||
// flip doorkeeper bit if not already
|
||||
if added := p.door.AddIfNotHas(key); !added {
|
||||
// increment count-min counter if doorkeeper bit is already set.
|
||||
p.freq.Increment(key)
|
||||
}
|
||||
p.incrs++
|
||||
if p.incrs >= p.resetAt {
|
||||
p.reset()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *tinyLFU) reset() {
|
||||
// Zero out incrs.
|
||||
p.incrs = 0
|
||||
// clears doorkeeper bits
|
||||
p.door.Clear()
|
||||
// halves count-min counters
|
||||
p.freq.Reset()
|
||||
}
|
||||
|
||||
// lruPolicy is different than the default policy in that it uses exact LRU
|
||||
// eviction rather than Sampled LFU eviction, which may be useful for certain
|
||||
// workloads (ARC-OLTP for example; LRU heavy workloads).
|
||||
//
|
||||
// TODO: - cost based eviction (multiple evictions for one new item, etc.)
|
||||
// - sampled LRU
|
||||
type lruPolicy struct {
|
||||
sync.Mutex
|
||||
admit *tinyLFU
|
||||
ptrs map[uint64]*lruItem
|
||||
vals *list.List
|
||||
maxCost int64
|
||||
room int64
|
||||
}
|
||||
|
||||
type lruItem struct {
|
||||
ptr *list.Element
|
||||
key uint64
|
||||
cost int64
|
||||
}
|
||||
|
||||
func newLRUPolicy(numCounters, maxCost int64) policy {
|
||||
return &lruPolicy{
|
||||
admit: newTinyLFU(numCounters),
|
||||
ptrs: make(map[uint64]*lruItem, maxCost),
|
||||
vals: list.New(),
|
||||
room: maxCost,
|
||||
maxCost: maxCost,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *lruPolicy) Push(keys []uint64) bool {
|
||||
if len(keys) == 0 {
|
||||
return true
|
||||
}
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
for _, key := range keys {
|
||||
// increment tinylfu counter
|
||||
p.admit.Increment(key)
|
||||
// move list item to front
|
||||
if val, ok := p.ptrs[key]; ok {
|
||||
// move accessed val to MRU position
|
||||
p.vals.MoveToFront(val.ptr)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *lruPolicy) Add(key uint64, cost int64) ([]*item, bool) {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
if cost > p.maxCost {
|
||||
return nil, false
|
||||
}
|
||||
if val, has := p.ptrs[key]; has {
|
||||
p.vals.MoveToFront(val.ptr)
|
||||
return nil, true
|
||||
}
|
||||
victims := make([]*item, 0)
|
||||
incHits := p.admit.Estimate(key)
|
||||
if p.room >= 0 {
|
||||
goto add
|
||||
}
|
||||
for p.room < 0 {
|
||||
lru := p.vals.Back()
|
||||
victim := lru.Value.(*lruItem)
|
||||
if incHits < p.admit.Estimate(victim.key) {
|
||||
return victims, false
|
||||
}
|
||||
// delete victim from metadata
|
||||
p.vals.Remove(victim.ptr)
|
||||
delete(p.ptrs, victim.key)
|
||||
victims = append(victims, &item{victim.key, nil, victim.cost})
|
||||
// adjust room
|
||||
p.room += victim.cost
|
||||
}
|
||||
add:
|
||||
item := &lruItem{key: key, cost: cost}
|
||||
item.ptr = p.vals.PushFront(item)
|
||||
p.ptrs[key] = item
|
||||
p.room -= cost
|
||||
return victims, true
|
||||
}
|
||||
|
||||
func (p *lruPolicy) Has(key uint64) bool {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
_, has := p.ptrs[key]
|
||||
return has
|
||||
}
|
||||
|
||||
func (p *lruPolicy) Del(key uint64) {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
if val, ok := p.ptrs[key]; ok {
|
||||
p.vals.Remove(val.ptr)
|
||||
delete(p.ptrs, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *lruPolicy) Cap() int64 {
|
||||
p.Lock()
|
||||
defer p.Unlock()
|
||||
return int64(p.vals.Len())
|
||||
}
|
||||
|
||||
// TODO
|
||||
func (p *lruPolicy) CollectMetrics(stats *metrics) {
|
||||
}
|
||||
145
vendor/github.com/dgraph-io/ristretto/ring.go
generated
vendored
Normal file
145
vendor/github.com/dgraph-io/ristretto/ring.go
generated
vendored
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package ristretto
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
ringLossy byte = iota
|
||||
ringLossless
|
||||
)
|
||||
|
||||
// ringConsumer is the user-defined object responsible for receiving and
|
||||
// processing items in batches when buffers are drained.
|
||||
type ringConsumer interface {
|
||||
Push([]uint64) bool
|
||||
}
|
||||
|
||||
// ringStripe is a singular ring buffer that is not concurrent safe.
|
||||
type ringStripe struct {
|
||||
consumer ringConsumer
|
||||
data []uint64
|
||||
capacity int
|
||||
busy int32
|
||||
}
|
||||
|
||||
func newRingStripe(config *ringConfig) *ringStripe {
|
||||
return &ringStripe{
|
||||
consumer: config.Consumer,
|
||||
data: make([]uint64, 0, config.Capacity),
|
||||
capacity: int(config.Capacity),
|
||||
}
|
||||
}
|
||||
|
||||
// Push appends an item in the ring buffer and drains (copies items and
|
||||
// sends to Consumer) if full.
|
||||
func (s *ringStripe) Push(item uint64) {
|
||||
s.data = append(s.data, item)
|
||||
// if we should drain
|
||||
if len(s.data) >= s.capacity {
|
||||
// Send elements to consumer. Create a new one.
|
||||
if s.consumer.Push(s.data) {
|
||||
s.data = make([]uint64, 0, s.capacity)
|
||||
} else {
|
||||
s.data = s.data[:0]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ringConfig is passed to newRingBuffer with parameters.
|
||||
type ringConfig struct {
|
||||
Consumer ringConsumer
|
||||
Stripes int64
|
||||
Capacity int64
|
||||
}
|
||||
|
||||
// ringBuffer stores multiple buffers (stripes) and distributes Pushed items
|
||||
// between them to lower contention.
|
||||
//
|
||||
// This implements the "batching" process described in the BP-Wrapper paper
|
||||
// (section III part A).
|
||||
type ringBuffer struct {
|
||||
stripes []*ringStripe
|
||||
pool *sync.Pool
|
||||
push func(*ringBuffer, uint64)
|
||||
rand int
|
||||
mask int
|
||||
}
|
||||
|
||||
// newRingBuffer returns a striped ring buffer. The Type can be either LOSSY or
|
||||
// LOSSLESS. LOSSY should provide better performance. The Consumer in ringConfig
|
||||
// will be called when individual stripes are full and need to drain their
|
||||
// elements.
|
||||
func newRingBuffer(ringType byte, config *ringConfig) *ringBuffer {
|
||||
if ringType == ringLossy {
|
||||
// LOSSY buffers use a very simple sync.Pool for concurrently reusing
|
||||
// stripes. We do lose some stripes due to GC (unheld items in sync.Pool
|
||||
// are cleared), but the performance gains generally outweigh the small
|
||||
// percentage of elements lost. The performance primarily comes from
|
||||
// low-level runtime functions used in the standard library that aren't
|
||||
// available to us (such as runtime_procPin()).
|
||||
return &ringBuffer{
|
||||
pool: &sync.Pool{
|
||||
New: func() interface{} { return newRingStripe(config) },
|
||||
},
|
||||
push: pushLossy,
|
||||
}
|
||||
}
|
||||
// begin LOSSLESS buffer handling
|
||||
//
|
||||
// unlike lossy, lossless manually handles all stripes
|
||||
stripes := make([]*ringStripe, config.Stripes)
|
||||
for i := range stripes {
|
||||
stripes[i] = newRingStripe(config)
|
||||
}
|
||||
return &ringBuffer{
|
||||
stripes: stripes,
|
||||
mask: int(config.Stripes - 1),
|
||||
rand: int(time.Now().UnixNano()), // random seed for picking stripes
|
||||
push: pushLossless,
|
||||
}
|
||||
}
|
||||
|
||||
// Push adds an element to one of the internal stripes and possibly drains if
|
||||
// the stripe becomes full.
|
||||
func (b *ringBuffer) Push(item uint64) {
|
||||
b.push(b, item)
|
||||
}
|
||||
|
||||
func pushLossy(b *ringBuffer, item uint64) {
|
||||
// reuse or create a new stripe
|
||||
stripe := b.pool.Get().(*ringStripe)
|
||||
stripe.Push(item)
|
||||
b.pool.Put(stripe)
|
||||
}
|
||||
|
||||
func pushLossless(b *ringBuffer, item uint64) {
|
||||
// try to find an available stripe
|
||||
for i := 0; ; i = (i + 1) & b.mask {
|
||||
if atomic.CompareAndSwapInt32(&b.stripes[i].busy, 0, 1) {
|
||||
// try to get exclusive lock on the stripe
|
||||
b.stripes[i].Push(item)
|
||||
// unlock
|
||||
atomic.StoreInt32(&b.stripes[i].busy, 0)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
189
vendor/github.com/dgraph-io/ristretto/sketch.go
generated
vendored
Normal file
189
vendor/github.com/dgraph-io/ristretto/sketch.go
generated
vendored
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// This package includes multiple probabalistic data structures needed for
|
||||
// admission/eviction metadata. Most are Counting Bloom Filter variations, but
|
||||
// a caching-specific feature that is also required is a "freshness" mechanism,
|
||||
// which basically serves as a "lifetime" process. This freshness mechanism
|
||||
// was described in the original TinyLFU paper [1], but other mechanisms may
|
||||
// be better suited for certain data distributions.
|
||||
//
|
||||
// [1]: https://arxiv.org/abs/1512.00727
|
||||
package ristretto
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// cmSketch is a Count-Min sketch implementation with 4-bit counters, heavily
|
||||
// based on Damian Gryski's CM4 [1].
|
||||
//
|
||||
// [1]: https://github.com/dgryski/go-tinylfu/blob/master/cm4.go
|
||||
type cmSketch struct {
|
||||
rows [cmDepth]cmRow
|
||||
mask uint32
|
||||
}
|
||||
|
||||
const (
|
||||
// cmDepth is the number of counter copies to store (think of it as rows)
|
||||
cmDepth = 4
|
||||
)
|
||||
|
||||
func newCmSketch(numCounters int64) *cmSketch {
|
||||
if numCounters == 0 {
|
||||
panic("cmSketch: bad numCounters")
|
||||
}
|
||||
// get the next power of 2 for better cache performance
|
||||
numCounters = next2Power(numCounters)
|
||||
// sketch with FNV-64a hashing algorithm
|
||||
sketch := &cmSketch{
|
||||
mask: uint32(numCounters - 1),
|
||||
}
|
||||
// initialize rows of counters
|
||||
for i := 0; i < cmDepth; i++ {
|
||||
sketch.rows[i] = newCmRow(numCounters)
|
||||
}
|
||||
return sketch
|
||||
}
|
||||
|
||||
// Increment increments the count(ers) for the specified key.
|
||||
func (s *cmSketch) Increment(hashed uint64) {
|
||||
l, r := uint32(hashed), uint32(hashed>>32)
|
||||
for i := range s.rows {
|
||||
// increment the counter on each row
|
||||
s.rows[i].increment((l + uint32(i)*r) & s.mask)
|
||||
}
|
||||
}
|
||||
|
||||
// Estimate returns the value of the specified key.
|
||||
func (s *cmSketch) Estimate(hashed uint64) int64 {
|
||||
l, r := uint32(hashed), uint32(hashed>>32)
|
||||
min := byte(255)
|
||||
for i := range s.rows {
|
||||
// find the smallest counter value from all the rows
|
||||
if v := s.rows[i].get((l + uint32(i)*r) & s.mask); v < min {
|
||||
min = v
|
||||
}
|
||||
}
|
||||
return int64(min)
|
||||
}
|
||||
|
||||
// Reset halves all counter values.
|
||||
func (s *cmSketch) Reset() {
|
||||
for _, r := range s.rows {
|
||||
r.reset()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *cmSketch) string() string {
|
||||
var state string
|
||||
for i := range s.rows {
|
||||
state += " [ "
|
||||
state += s.rows[i].string()
|
||||
state += " ]\n"
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
// cmRow is a row of bytes, with each byte holding two counters
|
||||
type cmRow []byte
|
||||
|
||||
func newCmRow(numCounters int64) cmRow {
|
||||
return make(cmRow, numCounters/2)
|
||||
}
|
||||
|
||||
func (r cmRow) get(n uint32) byte {
|
||||
return byte(r[n/2]>>((n&1)*4)) & 0x0f
|
||||
}
|
||||
|
||||
func (r cmRow) increment(n uint32) {
|
||||
// index of the counter
|
||||
i := n / 2
|
||||
// shift distance (even 0, odd 4)
|
||||
s := (n & 1) * 4
|
||||
// counter value
|
||||
v := (r[i] >> s) & 0x0f
|
||||
// only increment if not max value (overflow wrap is bad for LFU)
|
||||
if v < 15 {
|
||||
r[i] += 1 << s
|
||||
}
|
||||
}
|
||||
|
||||
func (r cmRow) reset() {
|
||||
// halve each counter
|
||||
for i := range r {
|
||||
r[i] = (r[i] >> 1) & 0x77
|
||||
}
|
||||
}
|
||||
|
||||
func (r cmRow) string() string {
|
||||
var state string
|
||||
for i := uint64(0); i < uint64(len(r)*2); i++ {
|
||||
state += fmt.Sprintf("%02d ", (r[(i/2)]>>((i&1)*4))&0x0f)
|
||||
}
|
||||
state = state[:len(state)-1]
|
||||
return state
|
||||
}
|
||||
|
||||
// next2Power rounds x up to the next power of 2, if it's not already one.
|
||||
func next2Power(x int64) int64 {
|
||||
x--
|
||||
x |= x >> 1
|
||||
x |= x >> 2
|
||||
x |= x >> 4
|
||||
x |= x >> 8
|
||||
x |= x >> 16
|
||||
x |= x >> 32
|
||||
x++
|
||||
return x
|
||||
}
|
||||
|
||||
/*
|
||||
// TODO
|
||||
//
|
||||
// Fingerprint Counting Bloom Filter (FP-CBF): lower false positive rates than
|
||||
// basic CBF with little added complexity.
|
||||
//
|
||||
// https://doi.org/10.1016/j.ipl.2015.11.002
|
||||
type FPCBF struct {
|
||||
}
|
||||
|
||||
func (c *FPCBF) Push(keys []ring.Element) {}
|
||||
func (c *FPCBF) Estimate(hashed int64) int64 { return 0 }
|
||||
|
||||
// TODO
|
||||
//
|
||||
// d-left Counting Bloom Filter: based on d-left hashing which allows for much
|
||||
// better space efficiency (usually saving a factor of 2 or more).
|
||||
//
|
||||
// https://link.springer.com/chapter/10.1007/11841036_61
|
||||
type DLCBF struct {
|
||||
}
|
||||
|
||||
func (c *DLCBF) Push(keys []ring.Element) {}
|
||||
func (c *DLCBF) Estimate(hashed int64) int64 { return 0 }
|
||||
|
||||
// TODO
|
||||
//
|
||||
// Bloom Clock: this might be a good route for keeping track of LRU information
|
||||
// in a space efficient, probabilistic manner.
|
||||
//
|
||||
// https://arxiv.org/abs/1905.13064
|
||||
type BC struct{}
|
||||
|
||||
func (c *BC) Push(keys []ring.Element) {}
|
||||
func (c *BC) Estimate(hashed int64) int64 { return 0 }
|
||||
*/
|
||||
120
vendor/github.com/dgraph-io/ristretto/store.go
generated
vendored
Normal file
120
vendor/github.com/dgraph-io/ristretto/store.go
generated
vendored
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package ristretto
|
||||
|
||||
import (
|
||||
"sync"
|
||||
)
|
||||
|
||||
// store is the interface fulfilled by all hash map implementations in this
|
||||
// file. Some hash map implementations are better suited for certain data
|
||||
// distributions than others, so this allows us to abstract that out for use
|
||||
// in Ristretto.
|
||||
//
|
||||
// Every store is safe for concurrent usage.
|
||||
type store interface {
|
||||
// Get returns the value associated with the key parameter.
|
||||
Get(uint64) (interface{}, bool)
|
||||
// Set adds the key-value pair to the Map or updates the value if it's
|
||||
// already present.
|
||||
Set(uint64, interface{})
|
||||
// Del deletes the key-value pair from the Map.
|
||||
Del(uint64)
|
||||
}
|
||||
|
||||
// newStore returns the default store implementation.
|
||||
func newStore() store {
|
||||
// return newSyncMap()
|
||||
return newShardedMap()
|
||||
}
|
||||
|
||||
type syncMap struct {
|
||||
*sync.Map
|
||||
}
|
||||
|
||||
func newSyncMap() store {
|
||||
return &syncMap{&sync.Map{}}
|
||||
}
|
||||
|
||||
func (m *syncMap) Get(key uint64) (interface{}, bool) {
|
||||
return m.Load(key)
|
||||
}
|
||||
|
||||
func (m *syncMap) Set(key uint64, value interface{}) {
|
||||
m.Store(key, value)
|
||||
}
|
||||
|
||||
func (m *syncMap) Del(key uint64) {
|
||||
m.Delete(key)
|
||||
}
|
||||
|
||||
const numShards uint64 = 256
|
||||
|
||||
type shardedMap struct {
|
||||
shards []*lockedMap
|
||||
}
|
||||
|
||||
func newShardedMap() *shardedMap {
|
||||
sm := &shardedMap{shards: make([]*lockedMap, int(numShards))}
|
||||
for i := range sm.shards {
|
||||
sm.shards[i] = newLockedMap()
|
||||
}
|
||||
return sm
|
||||
}
|
||||
|
||||
func (sm *shardedMap) Get(key uint64) (interface{}, bool) {
|
||||
idx := key % numShards
|
||||
return sm.shards[idx].Get(key)
|
||||
}
|
||||
|
||||
func (sm *shardedMap) Set(key uint64, value interface{}) {
|
||||
idx := key % numShards
|
||||
sm.shards[idx].Set(key, value)
|
||||
}
|
||||
|
||||
func (sm *shardedMap) Del(key uint64) {
|
||||
idx := key % numShards
|
||||
sm.shards[idx].Del(key)
|
||||
}
|
||||
|
||||
type lockedMap struct {
|
||||
sync.RWMutex
|
||||
data map[uint64]interface{}
|
||||
}
|
||||
|
||||
func newLockedMap() *lockedMap {
|
||||
return &lockedMap{data: make(map[uint64]interface{})}
|
||||
}
|
||||
|
||||
func (m *lockedMap) Get(key uint64) (interface{}, bool) {
|
||||
m.RLock()
|
||||
defer m.RUnlock()
|
||||
val, found := m.data[key]
|
||||
return val, found
|
||||
}
|
||||
|
||||
func (m *lockedMap) Set(key uint64, value interface{}) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
m.data[key] = value
|
||||
}
|
||||
|
||||
func (m *lockedMap) Del(key uint64) {
|
||||
m.Lock()
|
||||
defer m.Unlock()
|
||||
delete(m.data, key)
|
||||
}
|
||||
64
vendor/github.com/dgraph-io/ristretto/z/LICENSE
generated
vendored
Normal file
64
vendor/github.com/dgraph-io/ristretto/z/LICENSE
generated
vendored
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
bbloom.go
|
||||
|
||||
// The MIT License (MIT)
|
||||
// Copyright (c) 2014 Andreas Briese, eduToolbox@Bri-C GmbH, Sarstedt
|
||||
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
// this software and associated documentation files (the "Software"), to deal in
|
||||
// the Software without restriction, including without limitation the rights to
|
||||
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
// the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
rtutil.go
|
||||
|
||||
// MIT License
|
||||
|
||||
// Copyright (c) 2019 Ewan Chou
|
||||
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
Modifications:
|
||||
|
||||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
129
vendor/github.com/dgraph-io/ristretto/z/README.md
generated
vendored
Normal file
129
vendor/github.com/dgraph-io/ristretto/z/README.md
generated
vendored
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
## bbloom: a bitset Bloom filter for go/golang
|
||||
===
|
||||
|
||||
package implements a fast bloom filter with real 'bitset' and JSONMarshal/JSONUnmarshal to store/reload the Bloom filter.
|
||||
|
||||
NOTE: the package uses unsafe.Pointer to set and read the bits from the bitset. If you're uncomfortable with using the unsafe package, please consider using my bloom filter package at github.com/AndreasBriese/bloom
|
||||
|
||||
===
|
||||
|
||||
changelog 11/2015: new thread safe methods AddTS(), HasTS(), AddIfNotHasTS() following a suggestion from Srdjan Marinovic (github @a-little-srdjan), who used this to code a bloomfilter cache.
|
||||
|
||||
This bloom filter was developed to strengthen a website-log database and was tested and optimized for this log-entry mask: "2014/%02i/%02i %02i:%02i:%02i /info.html".
|
||||
Nonetheless bbloom should work with any other form of entries.
|
||||
|
||||
~~Hash function is a modified Berkeley DB sdbm hash (to optimize for smaller strings). sdbm http://www.cse.yorku.ca/~oz/hash.html~~
|
||||
|
||||
Found sipHash (SipHash-2-4, a fast short-input PRF created by Jean-Philippe Aumasson and Daniel J. Bernstein.) to be about as fast. sipHash had been ported by Dimtry Chestnyk to Go (github.com/dchest/siphash )
|
||||
|
||||
Minimum hashset size is: 512 ([4]uint64; will be set automatically).
|
||||
|
||||
###install
|
||||
|
||||
```sh
|
||||
go get github.com/AndreasBriese/bbloom
|
||||
```
|
||||
|
||||
###test
|
||||
+ change to folder ../bbloom
|
||||
+ create wordlist in file "words.txt" (you might use `python permut.py`)
|
||||
+ run 'go test -bench=.' within the folder
|
||||
|
||||
```go
|
||||
go test -bench=.
|
||||
```
|
||||
|
||||
~~If you've installed the GOCONVEY TDD-framework http://goconvey.co/ you can run the tests automatically.~~
|
||||
|
||||
using go's testing framework now (have in mind that the op timing is related to 65536 operations of Add, Has, AddIfNotHas respectively)
|
||||
|
||||
### usage
|
||||
|
||||
after installation add
|
||||
|
||||
```go
|
||||
import (
|
||||
...
|
||||
"github.com/AndreasBriese/bbloom"
|
||||
...
|
||||
)
|
||||
```
|
||||
|
||||
at your header. In the program use
|
||||
|
||||
```go
|
||||
// create a bloom filter for 65536 items and 1 % wrong-positive ratio
|
||||
bf := bbloom.New(float64(1<<16), float64(0.01))
|
||||
|
||||
// or
|
||||
// create a bloom filter with 650000 for 65536 items and 7 locs per hash explicitly
|
||||
// bf = bbloom.New(float64(650000), float64(7))
|
||||
// or
|
||||
bf = bbloom.New(650000.0, 7.0)
|
||||
|
||||
// add one item
|
||||
bf.Add([]byte("butter"))
|
||||
|
||||
// Number of elements added is exposed now
|
||||
// Note: ElemNum will not be included in JSON export (for compatability to older version)
|
||||
nOfElementsInFilter := bf.ElemNum
|
||||
|
||||
// check if item is in the filter
|
||||
isIn := bf.Has([]byte("butter")) // should be true
|
||||
isNotIn := bf.Has([]byte("Butter")) // should be false
|
||||
|
||||
// 'add only if item is new' to the bloomfilter
|
||||
added := bf.AddIfNotHas([]byte("butter")) // should be false because 'butter' is already in the set
|
||||
added = bf.AddIfNotHas([]byte("buTTer")) // should be true because 'buTTer' is new
|
||||
|
||||
// thread safe versions for concurrent use: AddTS, HasTS, AddIfNotHasTS
|
||||
// add one item
|
||||
bf.AddTS([]byte("peanutbutter"))
|
||||
// check if item is in the filter
|
||||
isIn = bf.HasTS([]byte("peanutbutter")) // should be true
|
||||
isNotIn = bf.HasTS([]byte("peanutButter")) // should be false
|
||||
// 'add only if item is new' to the bloomfilter
|
||||
added = bf.AddIfNotHasTS([]byte("butter")) // should be false because 'peanutbutter' is already in the set
|
||||
added = bf.AddIfNotHasTS([]byte("peanutbuTTer")) // should be true because 'penutbuTTer' is new
|
||||
|
||||
// convert to JSON ([]byte)
|
||||
Json := bf.JSONMarshal()
|
||||
|
||||
// bloomfilters Mutex is exposed for external un-/locking
|
||||
// i.e. mutex lock while doing JSON conversion
|
||||
bf.Mtx.Lock()
|
||||
Json = bf.JSONMarshal()
|
||||
bf.Mtx.Unlock()
|
||||
|
||||
// restore a bloom filter from storage
|
||||
bfNew := bbloom.JSONUnmarshal(Json)
|
||||
|
||||
isInNew := bfNew.Has([]byte("butter")) // should be true
|
||||
isNotInNew := bfNew.Has([]byte("Butter")) // should be false
|
||||
|
||||
```
|
||||
|
||||
to work with the bloom filter.
|
||||
|
||||
### why 'fast'?
|
||||
|
||||
It's about 3 times faster than William Fitzgeralds bitset bloom filter https://github.com/willf/bloom . And it is about so fast as my []bool set variant for Boom filters (see https://github.com/AndreasBriese/bloom ) but having a 8times smaller memory footprint:
|
||||
|
||||
|
||||
Bloom filter (filter size 524288, 7 hashlocs)
|
||||
github.com/AndreasBriese/bbloom 'Add' 65536 items (10 repetitions): 6595800 ns (100 ns/op)
|
||||
github.com/AndreasBriese/bbloom 'Has' 65536 items (10 repetitions): 5986600 ns (91 ns/op)
|
||||
github.com/AndreasBriese/bloom 'Add' 65536 items (10 repetitions): 6304684 ns (96 ns/op)
|
||||
github.com/AndreasBriese/bloom 'Has' 65536 items (10 repetitions): 6568663 ns (100 ns/op)
|
||||
|
||||
github.com/willf/bloom 'Add' 65536 items (10 repetitions): 24367224 ns (371 ns/op)
|
||||
github.com/willf/bloom 'Test' 65536 items (10 repetitions): 21881142 ns (333 ns/op)
|
||||
github.com/dataence/bloom/standard 'Add' 65536 items (10 repetitions): 23041644 ns (351 ns/op)
|
||||
github.com/dataence/bloom/standard 'Check' 65536 items (10 repetitions): 19153133 ns (292 ns/op)
|
||||
github.com/cabello/bloom 'Add' 65536 items (10 repetitions): 131921507 ns (2012 ns/op)
|
||||
github.com/cabello/bloom 'Contains' 65536 items (10 repetitions): 131108962 ns (2000 ns/op)
|
||||
|
||||
(on MBPro15 OSX10.8.5 i7 4Core 2.4Ghz)
|
||||
|
||||
|
||||
With 32bit bloom filters (bloom32) using modified sdbm, bloom32 does hashing with only 2 bit shifts, one xor and one substraction per byte. smdb is about as fast as fnv64a but gives less collisions with the dataset (see mask above). bloom.New(float64(10 * 1<<16),float64(7)) populated with 1<<16 random items from the dataset (see above) and tested against the rest results in less than 0.05% collisions.
|
||||
202
vendor/github.com/dgraph-io/ristretto/z/bbloom.go
generated
vendored
Normal file
202
vendor/github.com/dgraph-io/ristretto/z/bbloom.go
generated
vendored
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
// The MIT License (MIT)
|
||||
// Copyright (c) 2014 Andreas Briese, eduToolbox@Bri-C GmbH, Sarstedt
|
||||
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
// this software and associated documentation files (the "Software"), to deal in
|
||||
// the Software without restriction, including without limitation the rights to
|
||||
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
// the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
package z
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"math"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// helper
|
||||
var mask = []uint8{1, 2, 4, 8, 16, 32, 64, 128}
|
||||
|
||||
func getSize(ui64 uint64) (size uint64, exponent uint64) {
|
||||
if ui64 < uint64(512) {
|
||||
ui64 = uint64(512)
|
||||
}
|
||||
size = uint64(1)
|
||||
for size < ui64 {
|
||||
size <<= 1
|
||||
exponent++
|
||||
}
|
||||
return size, exponent
|
||||
}
|
||||
|
||||
func calcSizeByWrongPositives(numEntries, wrongs float64) (uint64, uint64) {
|
||||
size := -1 * numEntries * math.Log(wrongs) / math.Pow(float64(0.69314718056), 2)
|
||||
locs := math.Ceil(float64(0.69314718056) * size / numEntries)
|
||||
return uint64(size), uint64(locs)
|
||||
}
|
||||
|
||||
// NewBloomFilter returns a new bloomfilter.
|
||||
func NewBloomFilter(params ...float64) (bloomfilter *Bloom) {
|
||||
var entries, locs uint64
|
||||
if len(params) == 2 {
|
||||
if params[1] < 1 {
|
||||
entries, locs = calcSizeByWrongPositives(params[0], params[1])
|
||||
} else {
|
||||
entries, locs = uint64(params[0]), uint64(params[1])
|
||||
}
|
||||
} else {
|
||||
log.Fatal("usage: New(float64(number_of_entries), float64(number_of_hashlocations))" +
|
||||
" i.e. New(float64(1000), float64(3)) or New(float64(number_of_entries)," +
|
||||
" float64(number_of_hashlocations)) i.e. New(float64(1000), float64(0.03))")
|
||||
}
|
||||
size, exponent := getSize(uint64(entries))
|
||||
bloomfilter = &Bloom{
|
||||
sizeExp: exponent,
|
||||
size: size - 1,
|
||||
setLocs: locs,
|
||||
shift: 64 - exponent,
|
||||
}
|
||||
bloomfilter.Size(size)
|
||||
return bloomfilter
|
||||
}
|
||||
|
||||
// Bloom filter
|
||||
type Bloom struct {
|
||||
bitset []uint64
|
||||
ElemNum uint64
|
||||
sizeExp uint64
|
||||
size uint64
|
||||
setLocs uint64
|
||||
shift uint64
|
||||
}
|
||||
|
||||
// <--- http://www.cse.yorku.ca/~oz/hash.html
|
||||
// modified Berkeley DB Hash (32bit)
|
||||
// hash is casted to l, h = 16bit fragments
|
||||
// func (bl Bloom) absdbm(b *[]byte) (l, h uint64) {
|
||||
// hash := uint64(len(*b))
|
||||
// for _, c := range *b {
|
||||
// hash = uint64(c) + (hash << 6) + (hash << bl.sizeExp) - hash
|
||||
// }
|
||||
// h = hash >> bl.shift
|
||||
// l = hash << bl.shift >> bl.shift
|
||||
// return l, h
|
||||
// }
|
||||
|
||||
// Add adds hash of a key to the bloomfilter.
|
||||
func (bl *Bloom) Add(hash uint64) {
|
||||
h := hash >> bl.shift
|
||||
l := hash << bl.shift >> bl.shift
|
||||
for i := uint64(0); i < bl.setLocs; i++ {
|
||||
bl.Set((h + i*l) & bl.size)
|
||||
bl.ElemNum++
|
||||
}
|
||||
}
|
||||
|
||||
// Has checks if bit(s) for entry hash is/are set,
|
||||
// returns true if the hash was added to the Bloom Filter.
|
||||
func (bl Bloom) Has(hash uint64) bool {
|
||||
h := hash >> bl.shift
|
||||
l := hash << bl.shift >> bl.shift
|
||||
for i := uint64(0); i < bl.setLocs; i++ {
|
||||
switch bl.IsSet((h + i*l) & bl.size) {
|
||||
case false:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// AddIfNotHas only Adds hash, if it's not present in the bloomfilter.
|
||||
// Returns true if hash was added.
|
||||
// Returns false if hash was already registered in the bloomfilter.
|
||||
func (bl *Bloom) AddIfNotHas(hash uint64) bool {
|
||||
if bl.Has(hash) {
|
||||
return false
|
||||
}
|
||||
bl.Add(hash)
|
||||
return true
|
||||
}
|
||||
|
||||
// Size makes Bloom filter with as bitset of size sz.
|
||||
func (bl *Bloom) Size(sz uint64) {
|
||||
bl.bitset = make([]uint64, sz>>6)
|
||||
}
|
||||
|
||||
// Clear resets the Bloom filter.
|
||||
func (bl *Bloom) Clear() {
|
||||
for i := range bl.bitset {
|
||||
bl.bitset[i] = 0
|
||||
}
|
||||
}
|
||||
|
||||
// Set sets the bit[idx] of bitset.
|
||||
func (bl *Bloom) Set(idx uint64) {
|
||||
ptr := unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[idx>>6])) + uintptr((idx%64)>>3))
|
||||
*(*uint8)(ptr) |= mask[idx%8]
|
||||
}
|
||||
|
||||
// IsSet checks if bit[idx] of bitset is set, returns true/false.
|
||||
func (bl *Bloom) IsSet(idx uint64) bool {
|
||||
ptr := unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[idx>>6])) + uintptr((idx%64)>>3))
|
||||
r := ((*(*uint8)(ptr)) >> (idx % 8)) & 1
|
||||
return r == 1
|
||||
}
|
||||
|
||||
// bloomJSONImExport
|
||||
// Im/Export structure used by JSONMarshal / JSONUnmarshal
|
||||
type bloomJSONImExport struct {
|
||||
FilterSet []byte
|
||||
SetLocs uint64
|
||||
}
|
||||
|
||||
// NewWithBoolset takes a []byte slice and number of locs per entry,
|
||||
// returns the bloomfilter with a bitset populated according to the input []byte.
|
||||
func newWithBoolset(bs *[]byte, locs uint64) *Bloom {
|
||||
bloomfilter := NewBloomFilter(float64(len(*bs)<<3), float64(locs))
|
||||
for i, b := range *bs {
|
||||
*(*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&bloomfilter.bitset[0])) + uintptr(i))) = b
|
||||
}
|
||||
return bloomfilter
|
||||
}
|
||||
|
||||
// JSONUnmarshal takes JSON-Object (type bloomJSONImExport) as []bytes
|
||||
// returns bloom32 / bloom64 object.
|
||||
func JSONUnmarshal(dbData []byte) *Bloom {
|
||||
bloomImEx := bloomJSONImExport{}
|
||||
json.Unmarshal(dbData, &bloomImEx)
|
||||
buf := bytes.NewBuffer(bloomImEx.FilterSet)
|
||||
bs := buf.Bytes()
|
||||
bf := newWithBoolset(&bs, bloomImEx.SetLocs)
|
||||
return bf
|
||||
}
|
||||
|
||||
// JSONMarshal returns JSON-object (type bloomJSONImExport) as []byte.
|
||||
func (bl Bloom) JSONMarshal() []byte {
|
||||
bloomImEx := bloomJSONImExport{}
|
||||
bloomImEx.SetLocs = uint64(bl.setLocs)
|
||||
bloomImEx.FilterSet = make([]byte, len(bl.bitset)<<3)
|
||||
for i := range bloomImEx.FilterSet {
|
||||
bloomImEx.FilterSet[i] = *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[0])) +
|
||||
uintptr(i)))
|
||||
}
|
||||
data, err := json.Marshal(bloomImEx)
|
||||
if err != nil {
|
||||
log.Fatal("json.Marshal failed: ", err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
64
vendor/github.com/dgraph-io/ristretto/z/rtutil.go
generated
vendored
Normal file
64
vendor/github.com/dgraph-io/ristretto/z/rtutil.go
generated
vendored
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
// MIT License
|
||||
|
||||
// Copyright (c) 2019 Ewan Chou
|
||||
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
// SOFTWARE.
|
||||
|
||||
package z
|
||||
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// NanoTime returns the current time in nanoseconds from a monotonic clock.
|
||||
//go:linkname NanoTime runtime.nanotime
|
||||
func NanoTime() int64
|
||||
|
||||
// CPUTicks is a faster alternative to NanoTime to measure time duration.
|
||||
//go:linkname CPUTicks runtime.cputicks
|
||||
func CPUTicks() int64
|
||||
|
||||
type stringStruct struct {
|
||||
str unsafe.Pointer
|
||||
len int
|
||||
}
|
||||
|
||||
//go:noescape
|
||||
//go:linkname memhash runtime.memhash
|
||||
func memhash(p unsafe.Pointer, h, s uintptr) uintptr
|
||||
|
||||
// MemHash is the hash function used by go map, it utilizes available hardware instructions(behaves
|
||||
// as aeshash if aes instruction is available).
|
||||
// NOTE: The hash seed changes for every process. So, this cannot be used as a persistent hash.
|
||||
func MemHash(data []byte) uint64 {
|
||||
ss := (*stringStruct)(unsafe.Pointer(&data))
|
||||
return uint64(memhash(ss.str, 0, uintptr(ss.len)))
|
||||
}
|
||||
|
||||
// MemHashString is the hash function used by go map, it utilizes available hardware instructions
|
||||
// (behaves as aeshash if aes instruction is available).
|
||||
// NOTE: The hash seed changes for every process. So, this cannot be used as a persistent hash.
|
||||
func MemHashString(str string) uint64 {
|
||||
ss := (*stringStruct)(unsafe.Pointer(&str))
|
||||
return uint64(memhash(ss.str, 0, uintptr(ss.len)))
|
||||
}
|
||||
|
||||
// FastRand is a fast thread local random function.
|
||||
//go:linkname FastRand runtime.fastrand
|
||||
func FastRand() uint32
|
||||
0
vendor/github.com/dgraph-io/ristretto/z/rtutil.s
generated
vendored
Normal file
0
vendor/github.com/dgraph-io/ristretto/z/rtutil.s
generated
vendored
Normal file
41
vendor/github.com/dgraph-io/ristretto/z/z.go
generated
vendored
Normal file
41
vendor/github.com/dgraph-io/ristretto/z/z.go
generated
vendored
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
/*
|
||||
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package z
|
||||
|
||||
// KeyToHash interprets the type of key and converts it to a uint64 hash.
|
||||
func KeyToHash(key interface{}) uint64 {
|
||||
switch k := key.(type) {
|
||||
case uint64:
|
||||
return k
|
||||
case string:
|
||||
return MemHashString(k)
|
||||
case []byte:
|
||||
return MemHash(k)
|
||||
case byte:
|
||||
return MemHash([]byte{k})
|
||||
case int:
|
||||
return uint64(k)
|
||||
case int32:
|
||||
return uint64(k)
|
||||
case uint32:
|
||||
return uint64(k)
|
||||
case int64:
|
||||
return uint64(k)
|
||||
default:
|
||||
panic("Key type not supported")
|
||||
}
|
||||
}
|
||||
12
vendor/vendor.json
vendored
12
vendor/vendor.json
vendored
|
|
@ -74,6 +74,18 @@
|
|||
"revision": "504e848d77ea4752b3057b8fb46da0e7f746ccf3",
|
||||
"revisionTime": "2018-06-03T19:32:48Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "WBgnO6VroXxhpYS6kCiFS4jrW/c=",
|
||||
"path": "github.com/dgraph-io/ristretto",
|
||||
"revision": "ae326c3348e84b72d4f9db6b0ccb2f3551a0ed3e",
|
||||
"revisionTime": "2019-09-24T01:22:51Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "fp+NtBZPTvGfWtsxZWUQct/e0ZM=",
|
||||
"path": "github.com/dgraph-io/ristretto/z",
|
||||
"revision": "ae326c3348e84b72d4f9db6b0ccb2f3551a0ed3e",
|
||||
"revisionTime": "2019-09-24T01:22:51Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "Ad8LPSCP9HctFrmskh+S5HpHXcs=",
|
||||
"path": "github.com/docker/docker/pkg/reexec",
|
||||
|
|
|
|||
Loading…
Reference in a new issue