mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-20 10:52:25 +00:00
trie: try out ristretto cache
This commit is contained in:
Martin Holst Swende
parent
df89233b57
commit
fa166501ad
18 changed files with 2106 additions and 30 deletions
|
|
@ -17,7 +17,6 @@
|
||||||
package trie
|
package trie
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
|
@ -25,7 +24,6 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/allegro/bigcache"
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
|
|
@ -69,7 +67,7 @@ const secureKeyLength = 11 + 32
|
||||||
type Database struct {
|
type Database struct {
|
||||||
diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes
|
diskdb ethdb.KeyValueStore // Persistent storage for matured trie nodes
|
||||||
|
|
||||||
cleans *bigcache.BigCache // GC friendly memory cache of clean node RLPs
|
cleans *RistrettoCache // GC friendly memory cache of clean node RLPs
|
||||||
dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes
|
dirties map[common.Hash]*cachedNode // Data and references relationships of dirty nodes
|
||||||
oldest common.Hash // Oldest tracked node, flush-list head
|
oldest common.Hash // Oldest tracked node, flush-list head
|
||||||
newest common.Hash // Newest tracked node, flush-list tail
|
newest common.Hash // Newest tracked node, flush-list tail
|
||||||
|
|
@ -275,19 +273,6 @@ func expandNode(hash hashNode, n node) node {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// trienodeHasher is a struct to be used with BigCache, which uses a Hasher to
|
|
||||||
// determine which shard to place an entry into. It's not a cryptographic hash,
|
|
||||||
// just to provide a bit of anti-collision (default is FNV64a).
|
|
||||||
//
|
|
||||||
// Since trie keys are already hashes, we can just use the key directly to
|
|
||||||
// map shard id.
|
|
||||||
type trienodeHasher struct{}
|
|
||||||
|
|
||||||
// Sum64 implements the bigcache.Hasher interface.
|
|
||||||
func (t trienodeHasher) Sum64(key string) uint64 {
|
|
||||||
return binary.BigEndian.Uint64([]byte(key))
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewDatabase creates a new trie database to store ephemeral trie content before
|
// NewDatabase creates a new trie database to store ephemeral trie content before
|
||||||
// its written out to disk or garbage collected. No read cache is created, so all
|
// its written out to disk or garbage collected. No read cache is created, so all
|
||||||
// data retrievals will hit the underlying disk database.
|
// data retrievals will hit the underlying disk database.
|
||||||
|
|
@ -299,16 +284,9 @@ func NewDatabase(diskdb ethdb.KeyValueStore) *Database {
|
||||||
// before its written out to disk or garbage collected. It also acts as a read cache
|
// before its written out to disk or garbage collected. It also acts as a read cache
|
||||||
// for nodes loaded from disk.
|
// for nodes loaded from disk.
|
||||||
func NewDatabaseWithCache(diskdb ethdb.KeyValueStore, cache int) *Database {
|
func NewDatabaseWithCache(diskdb ethdb.KeyValueStore, cache int) *Database {
|
||||||
var cleans *bigcache.BigCache
|
var cleans *RistrettoCache
|
||||||
if cache > 0 {
|
if cache > 0 {
|
||||||
cleans, _ = bigcache.NewBigCache(bigcache.Config{
|
cleans, _ = NewRistrettoCache(cache)
|
||||||
Shards: 1024,
|
|
||||||
LifeWindow: time.Hour,
|
|
||||||
MaxEntriesInWindow: cache * 1024,
|
|
||||||
MaxEntrySize: 512,
|
|
||||||
HardMaxCacheSize: cache,
|
|
||||||
Hasher: trienodeHasher{},
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
return &Database{
|
return &Database{
|
||||||
diskdb: diskdb,
|
diskdb: diskdb,
|
||||||
|
|
@ -384,7 +362,7 @@ func (db *Database) insertPreimage(hash common.Hash, preimage []byte) {
|
||||||
func (db *Database) node(hash common.Hash) node {
|
func (db *Database) node(hash common.Hash) node {
|
||||||
// Retrieve the node from the clean cache if available
|
// Retrieve the node from the clean cache if available
|
||||||
if db.cleans != nil {
|
if db.cleans != nil {
|
||||||
if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil {
|
if enc, err := db.cleans.Get(hash); err == nil && enc != nil {
|
||||||
memcacheCleanHitMeter.Mark(1)
|
memcacheCleanHitMeter.Mark(1)
|
||||||
memcacheCleanReadMeter.Mark(int64(len(enc)))
|
memcacheCleanReadMeter.Mark(int64(len(enc)))
|
||||||
return mustDecodeNode(hash[:], enc)
|
return mustDecodeNode(hash[:], enc)
|
||||||
|
|
@ -404,7 +382,7 @@ func (db *Database) node(hash common.Hash) node {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
if db.cleans != nil {
|
if db.cleans != nil {
|
||||||
db.cleans.Set(string(hash[:]), enc)
|
db.cleans.Set(hash, enc)
|
||||||
memcacheCleanMissMeter.Mark(1)
|
memcacheCleanMissMeter.Mark(1)
|
||||||
memcacheCleanWriteMeter.Mark(int64(len(enc)))
|
memcacheCleanWriteMeter.Mark(int64(len(enc)))
|
||||||
}
|
}
|
||||||
|
|
@ -420,7 +398,7 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
|
||||||
}
|
}
|
||||||
// Retrieve the node from the clean cache if available
|
// Retrieve the node from the clean cache if available
|
||||||
if db.cleans != nil {
|
if db.cleans != nil {
|
||||||
if enc, err := db.cleans.Get(string(hash[:])); err == nil && enc != nil {
|
if enc, err := db.cleans.Get(hash); err == nil && enc != nil {
|
||||||
memcacheCleanHitMeter.Mark(1)
|
memcacheCleanHitMeter.Mark(1)
|
||||||
memcacheCleanReadMeter.Mark(int64(len(enc)))
|
memcacheCleanReadMeter.Mark(int64(len(enc)))
|
||||||
return enc, nil
|
return enc, nil
|
||||||
|
|
@ -438,7 +416,7 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) {
|
||||||
enc, err := db.diskdb.Get(hash[:])
|
enc, err := db.diskdb.Get(hash[:])
|
||||||
if err == nil && enc != nil {
|
if err == nil && enc != nil {
|
||||||
if db.cleans != nil {
|
if db.cleans != nil {
|
||||||
db.cleans.Set(string(hash[:]), enc)
|
db.cleans.Set(hash, enc)
|
||||||
memcacheCleanMissMeter.Mark(1)
|
memcacheCleanMissMeter.Mark(1)
|
||||||
memcacheCleanWriteMeter.Mark(int64(len(enc)))
|
memcacheCleanWriteMeter.Mark(int64(len(enc)))
|
||||||
}
|
}
|
||||||
|
|
@ -835,7 +813,7 @@ func (c *cleaner) Put(key []byte, rlp []byte) error {
|
||||||
}
|
}
|
||||||
// Move the flushed node into the clean cache to prevent insta-reloads
|
// Move the flushed node into the clean cache to prevent insta-reloads
|
||||||
if c.db.cleans != nil {
|
if c.db.cleans != nil {
|
||||||
c.db.cleans.Set(string(hash[:]), rlp)
|
c.db.cleans.Set(hash, rlp)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
64
trie/ristretto.go
Normal file
64
trie/ristretto.go
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
package trie
|
||||||
|
|
||||||
|
// Copyright 2019 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"github.com/dgraph-io/ristretto"
|
||||||
|
"github.com/ethereum/go-ethereum/common"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrMissingItem = errors.New("missing item")
|
||||||
|
|
||||||
|
type RistrettoCache struct {
|
||||||
|
cache *ristretto.Cache
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewRistrettoCache(capacity int) (*RistrettoCache, error) {
|
||||||
|
cache, err := ristretto.NewCache(&ristretto.Config{
|
||||||
|
NumCounters: int64(capacity * 10),
|
||||||
|
MaxCost: int64(capacity),
|
||||||
|
BufferItems: 64,
|
||||||
|
Metrics: false,
|
||||||
|
KeyToHash: func(key interface{}) uint64 {
|
||||||
|
h := key.(common.Hash)
|
||||||
|
return uint64(h[7]) | uint64(h[6])<<8 | uint64(h[5])<<16 | uint64(h[4])<<24 |
|
||||||
|
uint64(h[3])<<32 | uint64(h[2])<<40 | uint64(h[1])<<48 | uint64(h[0])<<56
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &RistrettoCache{
|
||||||
|
cache: cache,
|
||||||
|
}, nil
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *RistrettoCache) Get(key common.Hash) ([]byte, error) {
|
||||||
|
v, exist := c.cache.Get(key)
|
||||||
|
if exist {
|
||||||
|
return []byte(v.(string)), nil
|
||||||
|
}
|
||||||
|
return nil, ErrMissingItem
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *RistrettoCache) Set(key common.Hash, value []byte) error {
|
||||||
|
c.cache.Set(key, string(value), int64(len(value)))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
176
vendor/github.com/dgraph-io/ristretto/LICENSE
generated
vendored
Normal file
176
vendor/github.com/dgraph-io/ristretto/LICENSE
generated
vendored
Normal file
|
|
@ -0,0 +1,176 @@
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
104
vendor/github.com/dgraph-io/ristretto/README.md
generated
vendored
Normal file
104
vendor/github.com/dgraph-io/ristretto/README.md
generated
vendored
Normal file
|
|
@ -0,0 +1,104 @@
|
||||||
|
# Ristretto
|
||||||
|
|
||||||
|
[](https://godoc.org/github.com/dgraph-io/ristretto)
|
||||||
|
[](https://goreportcard.com/report/github.com/dgraph-io/ristretto)
|
||||||
|
|
||||||
|
Ristretto is a fast, concurrent cache library using a [TinyLFU](https://arxiv.org/abs/1512.00727)
|
||||||
|
admission policy and Sampled LFU eviction policy.
|
||||||
|
|
||||||
|
The motivation to build Ristretto comes from the need for a contention-free
|
||||||
|
cache in [Dgraph][].
|
||||||
|
|
||||||
|
[Dgraph]: https://github.com/dgraph-io/dgraph
|
||||||
|
|
||||||
|
## Example
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/dgraph-io/ristretto"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// create a cache instance
|
||||||
|
cache, err := ristretto.NewCache(&ristretto.Config{
|
||||||
|
NumCounters: 1000000 * 10,
|
||||||
|
MaxCost: 1000000,
|
||||||
|
BufferItems: 64,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// set a value
|
||||||
|
cache.Set("key", "value", 1)
|
||||||
|
|
||||||
|
// wait for value to pass through buffers
|
||||||
|
time.Sleep(time.Second / 100)
|
||||||
|
|
||||||
|
// get a value, given a key
|
||||||
|
value, found := cache.Get("key")
|
||||||
|
if !found {
|
||||||
|
panic("missing value")
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(value)
|
||||||
|
|
||||||
|
// delete a value, given a key
|
||||||
|
cache.Del("key")
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Benchmarks
|
||||||
|
|
||||||
|
The benchmarks can be found in https://github.com/dgraph-io/benchmarks/tree/master/cachebench/ristretto
|
||||||
|
|
||||||
|
### Hit Ratios
|
||||||
|
|
||||||
|
#### Search
|
||||||
|
|
||||||
|
This trace is described as "disk read accesses initiated by a large commercial
|
||||||
|
search engine in response to various web search requests."
|
||||||
|
|
||||||
|
.svg?sanitize=true)
|
||||||
|
|
||||||
|
#### Database
|
||||||
|
|
||||||
|
This trace is described as "a database server running at a commercial site
|
||||||
|
running an ERP application on top of a commercial database."
|
||||||
|
|
||||||
|
.svg?sanitize=true)
|
||||||
|
|
||||||
|
#### Looping
|
||||||
|
|
||||||
|
This trace demonstrates a looping access pattern.
|
||||||
|
|
||||||
|
.svg?sanitize=true)
|
||||||
|
|
||||||
|
#### CODASYL
|
||||||
|
|
||||||
|
This trace is described as "references to a CODASYL database for a one hour
|
||||||
|
period."
|
||||||
|
|
||||||
|
.svg?sanitize=true)
|
||||||
|
|
||||||
|
### Throughput
|
||||||
|
|
||||||
|
All throughput benchmarks were ran on an Intel Core i7-8700K (3.7GHz) with 16gb
|
||||||
|
of RAM.
|
||||||
|
|
||||||
|
#### Mixed
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#### Read
|
||||||
|
|
||||||
|
.svg?sanitize=true)
|
||||||
|
|
||||||
|
#### Write
|
||||||
|
|
||||||
|
.svg?sanitize=true)
|
||||||
358
vendor/github.com/dgraph-io/ristretto/cache.go
generated
vendored
Normal file
358
vendor/github.com/dgraph-io/ristretto/cache.go
generated
vendored
Normal file
|
|
@ -0,0 +1,358 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// Ristretto is a fast, fixed size, in-memory cache with a dual focus on
|
||||||
|
// throughput and hit ratio performance. You can easily add Ristretto to an
|
||||||
|
// existing system and keep the most valuable data where you need it.
|
||||||
|
package ristretto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
|
"github.com/dgraph-io/ristretto/z"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Cache is a thread-safe implementation of a hashmap with a TinyLFU admission
|
||||||
|
// policy and a Sampled LFU eviction policy. You can use the same Cache instance
|
||||||
|
// from as many goroutines as you want.
|
||||||
|
type Cache struct {
|
||||||
|
// store is the central concurrent hashmap where key-value items are stored
|
||||||
|
store store
|
||||||
|
// policy determines what gets let in to the cache and what gets kicked out
|
||||||
|
policy policy
|
||||||
|
// getBuf is a custom ring buffer implementation that gets pushed to when
|
||||||
|
// keys are read
|
||||||
|
getBuf *ringBuffer
|
||||||
|
// setBuf is a buffer allowing us to batch/drop Sets during times of high
|
||||||
|
// contention
|
||||||
|
setBuf chan *item
|
||||||
|
// stats contains a running log of important statistics like hits, misses,
|
||||||
|
// and dropped items
|
||||||
|
stats *metrics
|
||||||
|
// onEvict is called for item evictions
|
||||||
|
onEvict func(uint64, interface{}, int64)
|
||||||
|
// KeyToHash function is used to customize the key hashing algorithm.
|
||||||
|
// Each key will be hashed using the provided function. If keyToHash value
|
||||||
|
// is not set, the default keyToHash function is used.
|
||||||
|
keyToHash func(interface{}) uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config is passed to NewCache for creating new Cache instances.
|
||||||
|
type Config struct {
|
||||||
|
// NumCounters determines the number of counters (keys) to keep that hold
|
||||||
|
// access frequency information. It's generally a good idea to have more
|
||||||
|
// counters than the max cache capacity, as this will improve eviction
|
||||||
|
// accuracy and subsequent hit ratios.
|
||||||
|
//
|
||||||
|
// For example, if you expect your cache to hold 1,000,000 items when full,
|
||||||
|
// NumCounters should be 10,000,000 (10x). Each counter takes up 4 bits, so
|
||||||
|
// keeping 10,000,000 counters would require 5MB of memory.
|
||||||
|
NumCounters int64
|
||||||
|
// MaxCost can be considered as the cache capacity, in whatever units you
|
||||||
|
// choose to use.
|
||||||
|
//
|
||||||
|
// For example, if you want the cache to have a max capacity of 100MB, you
|
||||||
|
// would set MaxCost to 100,000,000 and pass an item's number of bytes as
|
||||||
|
// the `cost` parameter for calls to Set. If new items are accepted, the
|
||||||
|
// eviction process will take care of making room for the new item and not
|
||||||
|
// overflowing the MaxCost value.
|
||||||
|
MaxCost int64
|
||||||
|
// BufferItems determines the size of Get buffers.
|
||||||
|
//
|
||||||
|
// Unless you have a rare use case, using `64` as the BufferItems value
|
||||||
|
// results in good performance.
|
||||||
|
BufferItems int64
|
||||||
|
// Metrics determines whether cache statistics are kept during the cache's
|
||||||
|
// lifetime. There *is* some overhead to keeping statistics, so you should
|
||||||
|
// only set this flag to true when testing or throughput performance isn't a
|
||||||
|
// major factor.
|
||||||
|
Metrics bool
|
||||||
|
// OnEvict is called for every eviction and passes the hashed key, value,
|
||||||
|
// and cost to the function.
|
||||||
|
OnEvict func(key uint64, value interface{}, cost int64)
|
||||||
|
// KeyToHash function is used to customize the key hashing algorithm.
|
||||||
|
// Each key will be hashed using the provided function. If keyToHash value
|
||||||
|
// is not set, the default keyToHash function is used.
|
||||||
|
KeyToHash func(key interface{}) uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// item is passed to setBuf so items can eventually be added to the cache
|
||||||
|
type item struct {
|
||||||
|
key uint64
|
||||||
|
val interface{}
|
||||||
|
cost int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCache returns a new Cache instance and any configuration errors, if any.
|
||||||
|
func NewCache(config *Config) (*Cache, error) {
|
||||||
|
switch {
|
||||||
|
case config.NumCounters == 0:
|
||||||
|
return nil, errors.New("NumCounters can't be zero.")
|
||||||
|
case config.MaxCost == 0:
|
||||||
|
return nil, errors.New("MaxCost can't be zero.")
|
||||||
|
case config.BufferItems == 0:
|
||||||
|
return nil, errors.New("BufferItems can't be zero.")
|
||||||
|
}
|
||||||
|
policy := newPolicy(config.NumCounters, config.MaxCost)
|
||||||
|
cache := &Cache{
|
||||||
|
store: newStore(),
|
||||||
|
policy: policy,
|
||||||
|
getBuf: newRingBuffer(ringLossy, &ringConfig{
|
||||||
|
Consumer: policy,
|
||||||
|
Capacity: config.BufferItems,
|
||||||
|
}),
|
||||||
|
// TODO: size configuration for this? like BufferItems but for setBuf?
|
||||||
|
setBuf: make(chan *item, 32*1024),
|
||||||
|
onEvict: config.OnEvict,
|
||||||
|
keyToHash: config.KeyToHash,
|
||||||
|
}
|
||||||
|
if config.Metrics {
|
||||||
|
cache.collectMetrics()
|
||||||
|
}
|
||||||
|
// We can possibly make this configurable. But having 2 goroutines
|
||||||
|
// processing this seems sufficient for now.
|
||||||
|
//
|
||||||
|
// TODO: Allow a way to stop these goroutines.
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
go cache.processItems()
|
||||||
|
}
|
||||||
|
return cache, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns the value (if any) and a boolean representing whether the
|
||||||
|
// value was found or not. The value can be nil and the boolean can be true at
|
||||||
|
// the same time.
|
||||||
|
func (c *Cache) Get(key interface{}) (interface{}, bool) {
|
||||||
|
if c == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
hash := c.keyHash(key)
|
||||||
|
c.getBuf.Push(hash)
|
||||||
|
val, ok := c.store.Get(hash)
|
||||||
|
if ok {
|
||||||
|
c.stats.Add(hit, hash, 1)
|
||||||
|
} else {
|
||||||
|
c.stats.Add(miss, hash, 1)
|
||||||
|
}
|
||||||
|
return val, ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// keyHash generates the hash for a given key using the cutom keyToHash function, if provided.
|
||||||
|
// Otherwise it generates the hash using the z.KeyToHash funcion.
|
||||||
|
func (c *Cache) keyHash(key interface{}) uint64 {
|
||||||
|
if c.keyToHash != nil {
|
||||||
|
return c.keyToHash(key)
|
||||||
|
}
|
||||||
|
return z.KeyToHash(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set attempts to add the key-value item to the cache. If it returns false,
|
||||||
|
// then the Set was dropped and the key-value item isn't added to the cache. If
|
||||||
|
// it returns true, there's still a chance it could be dropped by the policy if
|
||||||
|
// its determined that the key-value item isn't worth keeping, but otherwise the
|
||||||
|
// item will be added and other items will be evicted in order to make room.
|
||||||
|
func (c *Cache) Set(key interface{}, val interface{}, cost int64) bool {
|
||||||
|
if c == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
hash := c.keyHash(key)
|
||||||
|
// TODO: Add a c.store.UpdateIfPresent here. This would catch any value updates and avoid having
|
||||||
|
// to push the key in setBuf.
|
||||||
|
|
||||||
|
// attempt to add the (possibly) new item to the setBuf where it will later
|
||||||
|
// be processed by the policy and evaluated
|
||||||
|
select {
|
||||||
|
case c.setBuf <- &item{key: hash, val: val, cost: cost}:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
// drop the set and avoid blocking
|
||||||
|
c.stats.Add(dropSets, hash, 1)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Add a public Update function, which would update a key only if present.
|
||||||
|
|
||||||
|
// Del deletes the key-value item from the cache if it exists.
|
||||||
|
func (c *Cache) Del(key interface{}) {
|
||||||
|
if c == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
hash := c.keyHash(key)
|
||||||
|
c.policy.Del(hash)
|
||||||
|
c.store.Del(hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close stops all goroutines and closes all channels.
|
||||||
|
func (c *Cache) Close() {}
|
||||||
|
|
||||||
|
// processItems is ran by goroutines processing the Set buffer.
|
||||||
|
func (c *Cache) processItems() {
|
||||||
|
for item := range c.setBuf {
|
||||||
|
victims, added := c.policy.Add(item.key, item.cost)
|
||||||
|
if added {
|
||||||
|
// item was accepted by the policy, so add to the hashmap
|
||||||
|
c.store.Set(item.key, item.val)
|
||||||
|
}
|
||||||
|
// delete victims that are no longer worthy of being in the cache
|
||||||
|
for _, victim := range victims {
|
||||||
|
// eviction callback
|
||||||
|
if c.onEvict != nil {
|
||||||
|
victim.val, _ = c.store.Get(victim.key)
|
||||||
|
c.onEvict(victim.key, victim.val, victim.cost)
|
||||||
|
}
|
||||||
|
// delete from hashmap
|
||||||
|
c.store.Del(victim.key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Cache) collectMetrics() {
|
||||||
|
c.stats = newMetrics()
|
||||||
|
c.policy.CollectMetrics(c.stats)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metrics returns statistics about cache performance.
|
||||||
|
func (c *Cache) Metrics() *metrics {
|
||||||
|
return c.stats
|
||||||
|
}
|
||||||
|
|
||||||
|
type metricType int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// The following 2 keep track of hits and misses.
|
||||||
|
hit = iota
|
||||||
|
miss
|
||||||
|
|
||||||
|
// The following 3 keep track of number of keys added, updated and evicted.
|
||||||
|
keyAdd
|
||||||
|
keyUpdate
|
||||||
|
keyEvict
|
||||||
|
|
||||||
|
// The following 2 keep track of cost of keys added and evicted.
|
||||||
|
costAdd
|
||||||
|
costEvict
|
||||||
|
|
||||||
|
// The following keep track of how many sets were dropped or rejected later.
|
||||||
|
dropSets
|
||||||
|
rejectSets
|
||||||
|
|
||||||
|
// The following 2 keep track of how many gets were kept and dropped on the floor.
|
||||||
|
dropGets
|
||||||
|
keepGets
|
||||||
|
|
||||||
|
// This should be the final enum. Other enums should be set before this.
|
||||||
|
doNotUse
|
||||||
|
)
|
||||||
|
|
||||||
|
func stringFor(t metricType) string {
|
||||||
|
switch t {
|
||||||
|
case hit:
|
||||||
|
return "hit"
|
||||||
|
case miss:
|
||||||
|
return "miss"
|
||||||
|
case keyAdd:
|
||||||
|
return "keys-added"
|
||||||
|
case keyUpdate:
|
||||||
|
return "keys-updated"
|
||||||
|
case keyEvict:
|
||||||
|
return "keys-evicted"
|
||||||
|
case costAdd:
|
||||||
|
return "cost-added"
|
||||||
|
case costEvict:
|
||||||
|
return "cost-evicted"
|
||||||
|
case dropSets:
|
||||||
|
return "sets-dropped"
|
||||||
|
case rejectSets:
|
||||||
|
return "sets-rejected" // by policy.
|
||||||
|
case dropGets:
|
||||||
|
return "gets-dropped"
|
||||||
|
case keepGets:
|
||||||
|
return "gets-kept"
|
||||||
|
default:
|
||||||
|
return "unidentified"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// metrics is the struct for hit ratio statistics. Note that there is some
|
||||||
|
// cost to maintaining the counters, so it's best to wrap Policies via the
|
||||||
|
// Recorder type when hit ratio analysis is needed.
|
||||||
|
type metrics struct {
|
||||||
|
all [doNotUse][]*uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMetrics() *metrics {
|
||||||
|
s := &metrics{}
|
||||||
|
for i := 0; i < doNotUse; i++ {
|
||||||
|
s.all[i] = make([]*uint64, 256)
|
||||||
|
slice := s.all[i]
|
||||||
|
for j := range slice {
|
||||||
|
slice[j] = new(uint64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *metrics) Add(t metricType, hash, delta uint64) {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
valp := p.all[t]
|
||||||
|
// Avoid false sharing by padding at least 64 bytes of space between two
|
||||||
|
// atomic counters which would be incremented.
|
||||||
|
idx := (hash % 25) * 10
|
||||||
|
atomic.AddUint64(valp[idx], delta)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *metrics) Get(t metricType) uint64 {
|
||||||
|
if p == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
valp := p.all[t]
|
||||||
|
var total uint64
|
||||||
|
for i := range valp {
|
||||||
|
total += atomic.LoadUint64(valp[i])
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *metrics) Ratio() float64 {
|
||||||
|
if p == nil {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
hits, misses := p.Get(hit), p.Get(miss)
|
||||||
|
if hits == 0 && misses == 0 {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
return float64(hits) / float64(hits+misses)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *metrics) String() string {
|
||||||
|
if p == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
for i := 0; i < doNotUse; i++ {
|
||||||
|
t := metricType(i)
|
||||||
|
fmt.Fprintf(&buf, "%s: %d ", stringFor(t), p.Get(t))
|
||||||
|
}
|
||||||
|
fmt.Fprintf(&buf, "gets-total: %d ", p.Get(hit)+p.Get(miss))
|
||||||
|
fmt.Fprintf(&buf, "hit-ratio: %.2f", p.Ratio())
|
||||||
|
return buf.String()
|
||||||
|
}
|
||||||
9
vendor/github.com/dgraph-io/ristretto/go.mod
generated
vendored
Normal file
9
vendor/github.com/dgraph-io/ristretto/go.mod
generated
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
module github.com/dgraph-io/ristretto
|
||||||
|
|
||||||
|
go 1.12
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||||
|
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2
|
||||||
|
github.com/stretchr/testify v1.3.0
|
||||||
|
)
|
||||||
10
vendor/github.com/dgraph-io/ristretto/go.sum
generated
vendored
Normal file
10
vendor/github.com/dgraph-io/ristretto/go.sum
generated
vendored
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
|
||||||
|
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||||
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
411
vendor/github.com/dgraph-io/ristretto/policy.go
generated
vendored
Normal file
411
vendor/github.com/dgraph-io/ristretto/policy.go
generated
vendored
Normal file
|
|
@ -0,0 +1,411 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package ristretto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"container/list"
|
||||||
|
"math"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/dgraph-io/ristretto/z"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// lfuSample is the number of items to sample when looking at eviction
|
||||||
|
// candidates. 5 seems to be the most optimal number [citation needed].
|
||||||
|
lfuSample = 5
|
||||||
|
)
|
||||||
|
|
||||||
|
// policy is the interface encapsulating eviction/admission behavior.
|
||||||
|
type policy interface {
|
||||||
|
ringConsumer
|
||||||
|
// Add attempts to Add the key-cost pair to the Policy. It returns a slice
|
||||||
|
// of evicted keys and a bool denoting whether or not the key-cost pair
|
||||||
|
// was added. If it returns true, the key should be stored in cache.
|
||||||
|
Add(uint64, int64) ([]*item, bool)
|
||||||
|
// Has returns true if the key exists in the Policy.
|
||||||
|
Has(uint64) bool
|
||||||
|
// Del deletes the key from the Policy.
|
||||||
|
Del(uint64)
|
||||||
|
// Cap returns the available capacity.
|
||||||
|
Cap() int64
|
||||||
|
// Optionally, set stats object to track how policy is performing.
|
||||||
|
CollectMetrics(stats *metrics)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPolicy(numCounters, maxCost int64) policy {
|
||||||
|
p := &defaultPolicy{
|
||||||
|
admit: newTinyLFU(numCounters),
|
||||||
|
evict: newSampledLFU(maxCost),
|
||||||
|
itemsCh: make(chan []uint64, 3),
|
||||||
|
}
|
||||||
|
// TODO: Add a way to stop the goroutine.
|
||||||
|
go p.processItems()
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// defaultPolicy is the default defaultPolicy, which is currently TinyLFU
|
||||||
|
// admission with sampledLFU eviction.
|
||||||
|
type defaultPolicy struct {
|
||||||
|
sync.Mutex
|
||||||
|
admit *tinyLFU
|
||||||
|
evict *sampledLFU
|
||||||
|
itemsCh chan []uint64
|
||||||
|
stats *metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *defaultPolicy) CollectMetrics(stats *metrics) {
|
||||||
|
p.stats = stats
|
||||||
|
p.evict.stats = stats
|
||||||
|
}
|
||||||
|
|
||||||
|
type policyPair struct {
|
||||||
|
key uint64
|
||||||
|
cost int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *defaultPolicy) processItems() {
|
||||||
|
for items := range p.itemsCh {
|
||||||
|
p.Lock()
|
||||||
|
p.admit.Push(items)
|
||||||
|
p.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *defaultPolicy) Push(keys []uint64) bool {
|
||||||
|
if len(keys) == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case p.itemsCh <- keys:
|
||||||
|
p.stats.Add(keepGets, keys[0], uint64(len(keys)))
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
p.stats.Add(dropGets, keys[0], uint64(len(keys)))
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *defaultPolicy) Add(key uint64, cost int64) ([]*item, bool) {
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
// can't add an item bigger than entire cache
|
||||||
|
if cost > p.evict.maxCost {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
// we don't need to go any further if the item is already in the cache
|
||||||
|
if has := p.evict.updateIfHas(key, cost); has {
|
||||||
|
return nil, true
|
||||||
|
}
|
||||||
|
// if we got this far, this key doesn't exist in the cache
|
||||||
|
//
|
||||||
|
// calculate the remaining room in the cache (usually bytes)
|
||||||
|
room := p.evict.roomLeft(cost)
|
||||||
|
if room >= 0 {
|
||||||
|
// there's enough room in the cache to store the new item without
|
||||||
|
// overflowing, so we can do that now and stop here
|
||||||
|
p.evict.add(key, cost)
|
||||||
|
return nil, true
|
||||||
|
}
|
||||||
|
// incHits is the hit count for the incoming item
|
||||||
|
incHits := p.admit.Estimate(key)
|
||||||
|
// sample is the eviction candidate pool to be filled via random sampling
|
||||||
|
//
|
||||||
|
// TODO: perhaps we should use a min heap here. Right now our time
|
||||||
|
// complexity is N for finding the min. Min heap should bring it down to
|
||||||
|
// O(lg N).
|
||||||
|
sample := make([]*policyPair, 0, lfuSample)
|
||||||
|
// as items are evicted they will be appended to victims
|
||||||
|
victims := make([]*item, 0)
|
||||||
|
// Delete victims until there's enough space or a minKey is found that has
|
||||||
|
// more hits than incoming item.
|
||||||
|
for ; room < 0; room = p.evict.roomLeft(cost) {
|
||||||
|
// fill up empty slots in sample
|
||||||
|
sample = p.evict.fillSample(sample)
|
||||||
|
// find minimally used item in sample
|
||||||
|
minKey, minHits, minId, minCost := uint64(0), int64(math.MaxInt64), 0, int64(0)
|
||||||
|
for i, pair := range sample {
|
||||||
|
// look up hit count for sample key
|
||||||
|
if hits := p.admit.Estimate(pair.key); hits < minHits {
|
||||||
|
minKey, minHits, minId, minCost = pair.key, hits, i, pair.cost
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// If the incoming item isn't worth keeping in the policy, reject.
|
||||||
|
if incHits < minHits {
|
||||||
|
p.stats.Add(rejectSets, key, 1)
|
||||||
|
return victims, false
|
||||||
|
}
|
||||||
|
// delete the victim from metadata
|
||||||
|
p.evict.del(minKey)
|
||||||
|
// delete the victim from sample
|
||||||
|
sample[minId] = sample[len(sample)-1]
|
||||||
|
sample = sample[:len(sample)-1]
|
||||||
|
// store victim in evicted victims slice
|
||||||
|
victims = append(victims, &item{minKey, nil, minCost})
|
||||||
|
}
|
||||||
|
p.evict.add(key, cost)
|
||||||
|
return victims, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *defaultPolicy) Has(key uint64) bool {
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
_, exists := p.evict.keyCosts[key]
|
||||||
|
return exists
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *defaultPolicy) Del(key uint64) {
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
p.evict.del(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *defaultPolicy) Cap() int64 {
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
return int64(p.evict.maxCost - p.evict.used)
|
||||||
|
}
|
||||||
|
|
||||||
|
// sampledLFU is an eviction helper storing key-cost pairs.
|
||||||
|
type sampledLFU struct {
|
||||||
|
keyCosts map[uint64]int64
|
||||||
|
maxCost int64
|
||||||
|
used int64
|
||||||
|
stats *metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSampledLFU(maxCost int64) *sampledLFU {
|
||||||
|
return &sampledLFU{
|
||||||
|
keyCosts: make(map[uint64]int64),
|
||||||
|
maxCost: maxCost,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *sampledLFU) roomLeft(cost int64) int64 {
|
||||||
|
return p.maxCost - (p.used + cost)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *sampledLFU) fillSample(in []*policyPair) []*policyPair {
|
||||||
|
if len(in) >= lfuSample {
|
||||||
|
return in
|
||||||
|
}
|
||||||
|
for key, cost := range p.keyCosts {
|
||||||
|
in = append(in, &policyPair{key, cost})
|
||||||
|
if len(in) >= lfuSample {
|
||||||
|
return in
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return in
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *sampledLFU) del(key uint64) {
|
||||||
|
cost, ok := p.keyCosts[key]
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p.stats.Add(keyEvict, key, 1)
|
||||||
|
p.stats.Add(costEvict, key, uint64(cost))
|
||||||
|
|
||||||
|
p.used -= cost
|
||||||
|
delete(p.keyCosts, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *sampledLFU) add(key uint64, cost int64) {
|
||||||
|
p.stats.Add(keyAdd, key, 1)
|
||||||
|
p.stats.Add(costAdd, key, uint64(cost))
|
||||||
|
|
||||||
|
p.keyCosts[key] = cost
|
||||||
|
p.used += cost
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Move this to the store itself. So, it can be used by public Set.
|
||||||
|
func (p *sampledLFU) updateIfHas(key uint64, cost int64) (updated bool) {
|
||||||
|
if prev, exists := p.keyCosts[key]; exists {
|
||||||
|
// Update the cost of the existing key. For simplicity, don't worry about evicting anything
|
||||||
|
// if the updated cost causes the size to grow beyond maxCost.
|
||||||
|
p.stats.Add(keyUpdate, key, 1)
|
||||||
|
p.used += cost - prev
|
||||||
|
p.keyCosts[key] = cost
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// tinyLFU is an admission helper that keeps track of access frequency using
|
||||||
|
// tiny (4-bit) counters in the form of a count-min sketch.
|
||||||
|
// tinyLFU is NOT thread safe.
|
||||||
|
type tinyLFU struct {
|
||||||
|
freq *cmSketch
|
||||||
|
door *z.Bloom
|
||||||
|
incrs int64
|
||||||
|
resetAt int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newTinyLFU(numCounters int64) *tinyLFU {
|
||||||
|
return &tinyLFU{
|
||||||
|
freq: newCmSketch(numCounters),
|
||||||
|
door: z.NewBloomFilter(float64(numCounters), 0.01),
|
||||||
|
resetAt: numCounters,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *tinyLFU) Push(keys []uint64) {
|
||||||
|
for _, key := range keys {
|
||||||
|
p.Increment(key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *tinyLFU) Estimate(key uint64) int64 {
|
||||||
|
hits := p.freq.Estimate(key)
|
||||||
|
if p.door.Has(key) {
|
||||||
|
hits += 1
|
||||||
|
}
|
||||||
|
return hits
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *tinyLFU) Increment(key uint64) {
|
||||||
|
// flip doorkeeper bit if not already
|
||||||
|
if added := p.door.AddIfNotHas(key); !added {
|
||||||
|
// increment count-min counter if doorkeeper bit is already set.
|
||||||
|
p.freq.Increment(key)
|
||||||
|
}
|
||||||
|
p.incrs++
|
||||||
|
if p.incrs >= p.resetAt {
|
||||||
|
p.reset()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *tinyLFU) reset() {
|
||||||
|
// Zero out incrs.
|
||||||
|
p.incrs = 0
|
||||||
|
// clears doorkeeper bits
|
||||||
|
p.door.Clear()
|
||||||
|
// halves count-min counters
|
||||||
|
p.freq.Reset()
|
||||||
|
}
|
||||||
|
|
||||||
|
// lruPolicy is different than the default policy in that it uses exact LRU
|
||||||
|
// eviction rather than Sampled LFU eviction, which may be useful for certain
|
||||||
|
// workloads (ARC-OLTP for example; LRU heavy workloads).
|
||||||
|
//
|
||||||
|
// TODO: - cost based eviction (multiple evictions for one new item, etc.)
|
||||||
|
// - sampled LRU
|
||||||
|
type lruPolicy struct {
|
||||||
|
sync.Mutex
|
||||||
|
admit *tinyLFU
|
||||||
|
ptrs map[uint64]*lruItem
|
||||||
|
vals *list.List
|
||||||
|
maxCost int64
|
||||||
|
room int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type lruItem struct {
|
||||||
|
ptr *list.Element
|
||||||
|
key uint64
|
||||||
|
cost int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newLRUPolicy(numCounters, maxCost int64) policy {
|
||||||
|
return &lruPolicy{
|
||||||
|
admit: newTinyLFU(numCounters),
|
||||||
|
ptrs: make(map[uint64]*lruItem, maxCost),
|
||||||
|
vals: list.New(),
|
||||||
|
room: maxCost,
|
||||||
|
maxCost: maxCost,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *lruPolicy) Push(keys []uint64) bool {
|
||||||
|
if len(keys) == 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
for _, key := range keys {
|
||||||
|
// increment tinylfu counter
|
||||||
|
p.admit.Increment(key)
|
||||||
|
// move list item to front
|
||||||
|
if val, ok := p.ptrs[key]; ok {
|
||||||
|
// move accessed val to MRU position
|
||||||
|
p.vals.MoveToFront(val.ptr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *lruPolicy) Add(key uint64, cost int64) ([]*item, bool) {
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
if cost > p.maxCost {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
if val, has := p.ptrs[key]; has {
|
||||||
|
p.vals.MoveToFront(val.ptr)
|
||||||
|
return nil, true
|
||||||
|
}
|
||||||
|
victims := make([]*item, 0)
|
||||||
|
incHits := p.admit.Estimate(key)
|
||||||
|
if p.room >= 0 {
|
||||||
|
goto add
|
||||||
|
}
|
||||||
|
for p.room < 0 {
|
||||||
|
lru := p.vals.Back()
|
||||||
|
victim := lru.Value.(*lruItem)
|
||||||
|
if incHits < p.admit.Estimate(victim.key) {
|
||||||
|
return victims, false
|
||||||
|
}
|
||||||
|
// delete victim from metadata
|
||||||
|
p.vals.Remove(victim.ptr)
|
||||||
|
delete(p.ptrs, victim.key)
|
||||||
|
victims = append(victims, &item{victim.key, nil, victim.cost})
|
||||||
|
// adjust room
|
||||||
|
p.room += victim.cost
|
||||||
|
}
|
||||||
|
add:
|
||||||
|
item := &lruItem{key: key, cost: cost}
|
||||||
|
item.ptr = p.vals.PushFront(item)
|
||||||
|
p.ptrs[key] = item
|
||||||
|
p.room -= cost
|
||||||
|
return victims, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *lruPolicy) Has(key uint64) bool {
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
_, has := p.ptrs[key]
|
||||||
|
return has
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *lruPolicy) Del(key uint64) {
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
if val, ok := p.ptrs[key]; ok {
|
||||||
|
p.vals.Remove(val.ptr)
|
||||||
|
delete(p.ptrs, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *lruPolicy) Cap() int64 {
|
||||||
|
p.Lock()
|
||||||
|
defer p.Unlock()
|
||||||
|
return int64(p.vals.Len())
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO
|
||||||
|
func (p *lruPolicy) CollectMetrics(stats *metrics) {
|
||||||
|
}
|
||||||
145
vendor/github.com/dgraph-io/ristretto/ring.go
generated
vendored
Normal file
145
vendor/github.com/dgraph-io/ristretto/ring.go
generated
vendored
Normal file
|
|
@ -0,0 +1,145 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package ristretto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ringLossy byte = iota
|
||||||
|
ringLossless
|
||||||
|
)
|
||||||
|
|
||||||
|
// ringConsumer is the user-defined object responsible for receiving and
|
||||||
|
// processing items in batches when buffers are drained.
|
||||||
|
type ringConsumer interface {
|
||||||
|
Push([]uint64) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ringStripe is a singular ring buffer that is not concurrent safe.
|
||||||
|
type ringStripe struct {
|
||||||
|
consumer ringConsumer
|
||||||
|
data []uint64
|
||||||
|
capacity int
|
||||||
|
busy int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func newRingStripe(config *ringConfig) *ringStripe {
|
||||||
|
return &ringStripe{
|
||||||
|
consumer: config.Consumer,
|
||||||
|
data: make([]uint64, 0, config.Capacity),
|
||||||
|
capacity: int(config.Capacity),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Push appends an item in the ring buffer and drains (copies items and
|
||||||
|
// sends to Consumer) if full.
|
||||||
|
func (s *ringStripe) Push(item uint64) {
|
||||||
|
s.data = append(s.data, item)
|
||||||
|
// if we should drain
|
||||||
|
if len(s.data) >= s.capacity {
|
||||||
|
// Send elements to consumer. Create a new one.
|
||||||
|
if s.consumer.Push(s.data) {
|
||||||
|
s.data = make([]uint64, 0, s.capacity)
|
||||||
|
} else {
|
||||||
|
s.data = s.data[:0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ringConfig is passed to newRingBuffer with parameters.
|
||||||
|
type ringConfig struct {
|
||||||
|
Consumer ringConsumer
|
||||||
|
Stripes int64
|
||||||
|
Capacity int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// ringBuffer stores multiple buffers (stripes) and distributes Pushed items
|
||||||
|
// between them to lower contention.
|
||||||
|
//
|
||||||
|
// This implements the "batching" process described in the BP-Wrapper paper
|
||||||
|
// (section III part A).
|
||||||
|
type ringBuffer struct {
|
||||||
|
stripes []*ringStripe
|
||||||
|
pool *sync.Pool
|
||||||
|
push func(*ringBuffer, uint64)
|
||||||
|
rand int
|
||||||
|
mask int
|
||||||
|
}
|
||||||
|
|
||||||
|
// newRingBuffer returns a striped ring buffer. The Type can be either LOSSY or
|
||||||
|
// LOSSLESS. LOSSY should provide better performance. The Consumer in ringConfig
|
||||||
|
// will be called when individual stripes are full and need to drain their
|
||||||
|
// elements.
|
||||||
|
func newRingBuffer(ringType byte, config *ringConfig) *ringBuffer {
|
||||||
|
if ringType == ringLossy {
|
||||||
|
// LOSSY buffers use a very simple sync.Pool for concurrently reusing
|
||||||
|
// stripes. We do lose some stripes due to GC (unheld items in sync.Pool
|
||||||
|
// are cleared), but the performance gains generally outweigh the small
|
||||||
|
// percentage of elements lost. The performance primarily comes from
|
||||||
|
// low-level runtime functions used in the standard library that aren't
|
||||||
|
// available to us (such as runtime_procPin()).
|
||||||
|
return &ringBuffer{
|
||||||
|
pool: &sync.Pool{
|
||||||
|
New: func() interface{} { return newRingStripe(config) },
|
||||||
|
},
|
||||||
|
push: pushLossy,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// begin LOSSLESS buffer handling
|
||||||
|
//
|
||||||
|
// unlike lossy, lossless manually handles all stripes
|
||||||
|
stripes := make([]*ringStripe, config.Stripes)
|
||||||
|
for i := range stripes {
|
||||||
|
stripes[i] = newRingStripe(config)
|
||||||
|
}
|
||||||
|
return &ringBuffer{
|
||||||
|
stripes: stripes,
|
||||||
|
mask: int(config.Stripes - 1),
|
||||||
|
rand: int(time.Now().UnixNano()), // random seed for picking stripes
|
||||||
|
push: pushLossless,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Push adds an element to one of the internal stripes and possibly drains if
|
||||||
|
// the stripe becomes full.
|
||||||
|
func (b *ringBuffer) Push(item uint64) {
|
||||||
|
b.push(b, item)
|
||||||
|
}
|
||||||
|
|
||||||
|
func pushLossy(b *ringBuffer, item uint64) {
|
||||||
|
// reuse or create a new stripe
|
||||||
|
stripe := b.pool.Get().(*ringStripe)
|
||||||
|
stripe.Push(item)
|
||||||
|
b.pool.Put(stripe)
|
||||||
|
}
|
||||||
|
|
||||||
|
func pushLossless(b *ringBuffer, item uint64) {
|
||||||
|
// try to find an available stripe
|
||||||
|
for i := 0; ; i = (i + 1) & b.mask {
|
||||||
|
if atomic.CompareAndSwapInt32(&b.stripes[i].busy, 0, 1) {
|
||||||
|
// try to get exclusive lock on the stripe
|
||||||
|
b.stripes[i].Push(item)
|
||||||
|
// unlock
|
||||||
|
atomic.StoreInt32(&b.stripes[i].busy, 0)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
189
vendor/github.com/dgraph-io/ristretto/sketch.go
generated
vendored
Normal file
189
vendor/github.com/dgraph-io/ristretto/sketch.go
generated
vendored
Normal file
|
|
@ -0,0 +1,189 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// This package includes multiple probabalistic data structures needed for
|
||||||
|
// admission/eviction metadata. Most are Counting Bloom Filter variations, but
|
||||||
|
// a caching-specific feature that is also required is a "freshness" mechanism,
|
||||||
|
// which basically serves as a "lifetime" process. This freshness mechanism
|
||||||
|
// was described in the original TinyLFU paper [1], but other mechanisms may
|
||||||
|
// be better suited for certain data distributions.
|
||||||
|
//
|
||||||
|
// [1]: https://arxiv.org/abs/1512.00727
|
||||||
|
package ristretto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cmSketch is a Count-Min sketch implementation with 4-bit counters, heavily
|
||||||
|
// based on Damian Gryski's CM4 [1].
|
||||||
|
//
|
||||||
|
// [1]: https://github.com/dgryski/go-tinylfu/blob/master/cm4.go
|
||||||
|
type cmSketch struct {
|
||||||
|
rows [cmDepth]cmRow
|
||||||
|
mask uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// cmDepth is the number of counter copies to store (think of it as rows)
|
||||||
|
cmDepth = 4
|
||||||
|
)
|
||||||
|
|
||||||
|
func newCmSketch(numCounters int64) *cmSketch {
|
||||||
|
if numCounters == 0 {
|
||||||
|
panic("cmSketch: bad numCounters")
|
||||||
|
}
|
||||||
|
// get the next power of 2 for better cache performance
|
||||||
|
numCounters = next2Power(numCounters)
|
||||||
|
// sketch with FNV-64a hashing algorithm
|
||||||
|
sketch := &cmSketch{
|
||||||
|
mask: uint32(numCounters - 1),
|
||||||
|
}
|
||||||
|
// initialize rows of counters
|
||||||
|
for i := 0; i < cmDepth; i++ {
|
||||||
|
sketch.rows[i] = newCmRow(numCounters)
|
||||||
|
}
|
||||||
|
return sketch
|
||||||
|
}
|
||||||
|
|
||||||
|
// Increment increments the count(ers) for the specified key.
|
||||||
|
func (s *cmSketch) Increment(hashed uint64) {
|
||||||
|
l, r := uint32(hashed), uint32(hashed>>32)
|
||||||
|
for i := range s.rows {
|
||||||
|
// increment the counter on each row
|
||||||
|
s.rows[i].increment((l + uint32(i)*r) & s.mask)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Estimate returns the value of the specified key.
|
||||||
|
func (s *cmSketch) Estimate(hashed uint64) int64 {
|
||||||
|
l, r := uint32(hashed), uint32(hashed>>32)
|
||||||
|
min := byte(255)
|
||||||
|
for i := range s.rows {
|
||||||
|
// find the smallest counter value from all the rows
|
||||||
|
if v := s.rows[i].get((l + uint32(i)*r) & s.mask); v < min {
|
||||||
|
min = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return int64(min)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset halves all counter values.
|
||||||
|
func (s *cmSketch) Reset() {
|
||||||
|
for _, r := range s.rows {
|
||||||
|
r.reset()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *cmSketch) string() string {
|
||||||
|
var state string
|
||||||
|
for i := range s.rows {
|
||||||
|
state += " [ "
|
||||||
|
state += s.rows[i].string()
|
||||||
|
state += " ]\n"
|
||||||
|
}
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
|
||||||
|
// cmRow is a row of bytes, with each byte holding two counters
|
||||||
|
type cmRow []byte
|
||||||
|
|
||||||
|
func newCmRow(numCounters int64) cmRow {
|
||||||
|
return make(cmRow, numCounters/2)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r cmRow) get(n uint32) byte {
|
||||||
|
return byte(r[n/2]>>((n&1)*4)) & 0x0f
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r cmRow) increment(n uint32) {
|
||||||
|
// index of the counter
|
||||||
|
i := n / 2
|
||||||
|
// shift distance (even 0, odd 4)
|
||||||
|
s := (n & 1) * 4
|
||||||
|
// counter value
|
||||||
|
v := (r[i] >> s) & 0x0f
|
||||||
|
// only increment if not max value (overflow wrap is bad for LFU)
|
||||||
|
if v < 15 {
|
||||||
|
r[i] += 1 << s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r cmRow) reset() {
|
||||||
|
// halve each counter
|
||||||
|
for i := range r {
|
||||||
|
r[i] = (r[i] >> 1) & 0x77
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r cmRow) string() string {
|
||||||
|
var state string
|
||||||
|
for i := uint64(0); i < uint64(len(r)*2); i++ {
|
||||||
|
state += fmt.Sprintf("%02d ", (r[(i/2)]>>((i&1)*4))&0x0f)
|
||||||
|
}
|
||||||
|
state = state[:len(state)-1]
|
||||||
|
return state
|
||||||
|
}
|
||||||
|
|
||||||
|
// next2Power rounds x up to the next power of 2, if it's not already one.
|
||||||
|
func next2Power(x int64) int64 {
|
||||||
|
x--
|
||||||
|
x |= x >> 1
|
||||||
|
x |= x >> 2
|
||||||
|
x |= x >> 4
|
||||||
|
x |= x >> 8
|
||||||
|
x |= x >> 16
|
||||||
|
x |= x >> 32
|
||||||
|
x++
|
||||||
|
return x
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
// TODO
|
||||||
|
//
|
||||||
|
// Fingerprint Counting Bloom Filter (FP-CBF): lower false positive rates than
|
||||||
|
// basic CBF with little added complexity.
|
||||||
|
//
|
||||||
|
// https://doi.org/10.1016/j.ipl.2015.11.002
|
||||||
|
type FPCBF struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *FPCBF) Push(keys []ring.Element) {}
|
||||||
|
func (c *FPCBF) Estimate(hashed int64) int64 { return 0 }
|
||||||
|
|
||||||
|
// TODO
|
||||||
|
//
|
||||||
|
// d-left Counting Bloom Filter: based on d-left hashing which allows for much
|
||||||
|
// better space efficiency (usually saving a factor of 2 or more).
|
||||||
|
//
|
||||||
|
// https://link.springer.com/chapter/10.1007/11841036_61
|
||||||
|
type DLCBF struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *DLCBF) Push(keys []ring.Element) {}
|
||||||
|
func (c *DLCBF) Estimate(hashed int64) int64 { return 0 }
|
||||||
|
|
||||||
|
// TODO
|
||||||
|
//
|
||||||
|
// Bloom Clock: this might be a good route for keeping track of LRU information
|
||||||
|
// in a space efficient, probabilistic manner.
|
||||||
|
//
|
||||||
|
// https://arxiv.org/abs/1905.13064
|
||||||
|
type BC struct{}
|
||||||
|
|
||||||
|
func (c *BC) Push(keys []ring.Element) {}
|
||||||
|
func (c *BC) Estimate(hashed int64) int64 { return 0 }
|
||||||
|
*/
|
||||||
120
vendor/github.com/dgraph-io/ristretto/store.go
generated
vendored
Normal file
120
vendor/github.com/dgraph-io/ristretto/store.go
generated
vendored
Normal file
|
|
@ -0,0 +1,120 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package ristretto
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// store is the interface fulfilled by all hash map implementations in this
|
||||||
|
// file. Some hash map implementations are better suited for certain data
|
||||||
|
// distributions than others, so this allows us to abstract that out for use
|
||||||
|
// in Ristretto.
|
||||||
|
//
|
||||||
|
// Every store is safe for concurrent usage.
|
||||||
|
type store interface {
|
||||||
|
// Get returns the value associated with the key parameter.
|
||||||
|
Get(uint64) (interface{}, bool)
|
||||||
|
// Set adds the key-value pair to the Map or updates the value if it's
|
||||||
|
// already present.
|
||||||
|
Set(uint64, interface{})
|
||||||
|
// Del deletes the key-value pair from the Map.
|
||||||
|
Del(uint64)
|
||||||
|
}
|
||||||
|
|
||||||
|
// newStore returns the default store implementation.
|
||||||
|
func newStore() store {
|
||||||
|
// return newSyncMap()
|
||||||
|
return newShardedMap()
|
||||||
|
}
|
||||||
|
|
||||||
|
type syncMap struct {
|
||||||
|
*sync.Map
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSyncMap() store {
|
||||||
|
return &syncMap{&sync.Map{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *syncMap) Get(key uint64) (interface{}, bool) {
|
||||||
|
return m.Load(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *syncMap) Set(key uint64, value interface{}) {
|
||||||
|
m.Store(key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *syncMap) Del(key uint64) {
|
||||||
|
m.Delete(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
const numShards uint64 = 256
|
||||||
|
|
||||||
|
type shardedMap struct {
|
||||||
|
shards []*lockedMap
|
||||||
|
}
|
||||||
|
|
||||||
|
func newShardedMap() *shardedMap {
|
||||||
|
sm := &shardedMap{shards: make([]*lockedMap, int(numShards))}
|
||||||
|
for i := range sm.shards {
|
||||||
|
sm.shards[i] = newLockedMap()
|
||||||
|
}
|
||||||
|
return sm
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sm *shardedMap) Get(key uint64) (interface{}, bool) {
|
||||||
|
idx := key % numShards
|
||||||
|
return sm.shards[idx].Get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sm *shardedMap) Set(key uint64, value interface{}) {
|
||||||
|
idx := key % numShards
|
||||||
|
sm.shards[idx].Set(key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sm *shardedMap) Del(key uint64) {
|
||||||
|
idx := key % numShards
|
||||||
|
sm.shards[idx].Del(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
type lockedMap struct {
|
||||||
|
sync.RWMutex
|
||||||
|
data map[uint64]interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func newLockedMap() *lockedMap {
|
||||||
|
return &lockedMap{data: make(map[uint64]interface{})}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *lockedMap) Get(key uint64) (interface{}, bool) {
|
||||||
|
m.RLock()
|
||||||
|
defer m.RUnlock()
|
||||||
|
val, found := m.data[key]
|
||||||
|
return val, found
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *lockedMap) Set(key uint64, value interface{}) {
|
||||||
|
m.Lock()
|
||||||
|
defer m.Unlock()
|
||||||
|
m.data[key] = value
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *lockedMap) Del(key uint64) {
|
||||||
|
m.Lock()
|
||||||
|
defer m.Unlock()
|
||||||
|
delete(m.data, key)
|
||||||
|
}
|
||||||
64
vendor/github.com/dgraph-io/ristretto/z/LICENSE
generated
vendored
Normal file
64
vendor/github.com/dgraph-io/ristretto/z/LICENSE
generated
vendored
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
bbloom.go
|
||||||
|
|
||||||
|
// The MIT License (MIT)
|
||||||
|
// Copyright (c) 2014 Andreas Briese, eduToolbox@Bri-C GmbH, Sarstedt
|
||||||
|
|
||||||
|
// Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
// this software and associated documentation files (the "Software"), to deal in
|
||||||
|
// the Software without restriction, including without limitation the rights to
|
||||||
|
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||||
|
// the Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
// subject to the following conditions:
|
||||||
|
|
||||||
|
// The above copyright notice and this permission notice shall be included in all
|
||||||
|
// copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||||
|
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||||
|
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
rtutil.go
|
||||||
|
|
||||||
|
// MIT License
|
||||||
|
|
||||||
|
// Copyright (c) 2019 Ewan Chou
|
||||||
|
|
||||||
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
// of this software and associated documentation files (the "Software"), to deal
|
||||||
|
// in the Software without restriction, including without limitation the rights
|
||||||
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
// copies of the Software, and to permit persons to whom the Software is
|
||||||
|
// furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
// The above copyright notice and this permission notice shall be included in all
|
||||||
|
// copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
// SOFTWARE.
|
||||||
|
|
||||||
|
Modifications:
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
129
vendor/github.com/dgraph-io/ristretto/z/README.md
generated
vendored
Normal file
129
vendor/github.com/dgraph-io/ristretto/z/README.md
generated
vendored
Normal file
|
|
@ -0,0 +1,129 @@
|
||||||
|
## bbloom: a bitset Bloom filter for go/golang
|
||||||
|
===
|
||||||
|
|
||||||
|
package implements a fast bloom filter with real 'bitset' and JSONMarshal/JSONUnmarshal to store/reload the Bloom filter.
|
||||||
|
|
||||||
|
NOTE: the package uses unsafe.Pointer to set and read the bits from the bitset. If you're uncomfortable with using the unsafe package, please consider using my bloom filter package at github.com/AndreasBriese/bloom
|
||||||
|
|
||||||
|
===
|
||||||
|
|
||||||
|
changelog 11/2015: new thread safe methods AddTS(), HasTS(), AddIfNotHasTS() following a suggestion from Srdjan Marinovic (github @a-little-srdjan), who used this to code a bloomfilter cache.
|
||||||
|
|
||||||
|
This bloom filter was developed to strengthen a website-log database and was tested and optimized for this log-entry mask: "2014/%02i/%02i %02i:%02i:%02i /info.html".
|
||||||
|
Nonetheless bbloom should work with any other form of entries.
|
||||||
|
|
||||||
|
~~Hash function is a modified Berkeley DB sdbm hash (to optimize for smaller strings). sdbm http://www.cse.yorku.ca/~oz/hash.html~~
|
||||||
|
|
||||||
|
Found sipHash (SipHash-2-4, a fast short-input PRF created by Jean-Philippe Aumasson and Daniel J. Bernstein.) to be about as fast. sipHash had been ported by Dimtry Chestnyk to Go (github.com/dchest/siphash )
|
||||||
|
|
||||||
|
Minimum hashset size is: 512 ([4]uint64; will be set automatically).
|
||||||
|
|
||||||
|
###install
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get github.com/AndreasBriese/bbloom
|
||||||
|
```
|
||||||
|
|
||||||
|
###test
|
||||||
|
+ change to folder ../bbloom
|
||||||
|
+ create wordlist in file "words.txt" (you might use `python permut.py`)
|
||||||
|
+ run 'go test -bench=.' within the folder
|
||||||
|
|
||||||
|
```go
|
||||||
|
go test -bench=.
|
||||||
|
```
|
||||||
|
|
||||||
|
~~If you've installed the GOCONVEY TDD-framework http://goconvey.co/ you can run the tests automatically.~~
|
||||||
|
|
||||||
|
using go's testing framework now (have in mind that the op timing is related to 65536 operations of Add, Has, AddIfNotHas respectively)
|
||||||
|
|
||||||
|
### usage
|
||||||
|
|
||||||
|
after installation add
|
||||||
|
|
||||||
|
```go
|
||||||
|
import (
|
||||||
|
...
|
||||||
|
"github.com/AndreasBriese/bbloom"
|
||||||
|
...
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
at your header. In the program use
|
||||||
|
|
||||||
|
```go
|
||||||
|
// create a bloom filter for 65536 items and 1 % wrong-positive ratio
|
||||||
|
bf := bbloom.New(float64(1<<16), float64(0.01))
|
||||||
|
|
||||||
|
// or
|
||||||
|
// create a bloom filter with 650000 for 65536 items and 7 locs per hash explicitly
|
||||||
|
// bf = bbloom.New(float64(650000), float64(7))
|
||||||
|
// or
|
||||||
|
bf = bbloom.New(650000.0, 7.0)
|
||||||
|
|
||||||
|
// add one item
|
||||||
|
bf.Add([]byte("butter"))
|
||||||
|
|
||||||
|
// Number of elements added is exposed now
|
||||||
|
// Note: ElemNum will not be included in JSON export (for compatability to older version)
|
||||||
|
nOfElementsInFilter := bf.ElemNum
|
||||||
|
|
||||||
|
// check if item is in the filter
|
||||||
|
isIn := bf.Has([]byte("butter")) // should be true
|
||||||
|
isNotIn := bf.Has([]byte("Butter")) // should be false
|
||||||
|
|
||||||
|
// 'add only if item is new' to the bloomfilter
|
||||||
|
added := bf.AddIfNotHas([]byte("butter")) // should be false because 'butter' is already in the set
|
||||||
|
added = bf.AddIfNotHas([]byte("buTTer")) // should be true because 'buTTer' is new
|
||||||
|
|
||||||
|
// thread safe versions for concurrent use: AddTS, HasTS, AddIfNotHasTS
|
||||||
|
// add one item
|
||||||
|
bf.AddTS([]byte("peanutbutter"))
|
||||||
|
// check if item is in the filter
|
||||||
|
isIn = bf.HasTS([]byte("peanutbutter")) // should be true
|
||||||
|
isNotIn = bf.HasTS([]byte("peanutButter")) // should be false
|
||||||
|
// 'add only if item is new' to the bloomfilter
|
||||||
|
added = bf.AddIfNotHasTS([]byte("butter")) // should be false because 'peanutbutter' is already in the set
|
||||||
|
added = bf.AddIfNotHasTS([]byte("peanutbuTTer")) // should be true because 'penutbuTTer' is new
|
||||||
|
|
||||||
|
// convert to JSON ([]byte)
|
||||||
|
Json := bf.JSONMarshal()
|
||||||
|
|
||||||
|
// bloomfilters Mutex is exposed for external un-/locking
|
||||||
|
// i.e. mutex lock while doing JSON conversion
|
||||||
|
bf.Mtx.Lock()
|
||||||
|
Json = bf.JSONMarshal()
|
||||||
|
bf.Mtx.Unlock()
|
||||||
|
|
||||||
|
// restore a bloom filter from storage
|
||||||
|
bfNew := bbloom.JSONUnmarshal(Json)
|
||||||
|
|
||||||
|
isInNew := bfNew.Has([]byte("butter")) // should be true
|
||||||
|
isNotInNew := bfNew.Has([]byte("Butter")) // should be false
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
to work with the bloom filter.
|
||||||
|
|
||||||
|
### why 'fast'?
|
||||||
|
|
||||||
|
It's about 3 times faster than William Fitzgeralds bitset bloom filter https://github.com/willf/bloom . And it is about so fast as my []bool set variant for Boom filters (see https://github.com/AndreasBriese/bloom ) but having a 8times smaller memory footprint:
|
||||||
|
|
||||||
|
|
||||||
|
Bloom filter (filter size 524288, 7 hashlocs)
|
||||||
|
github.com/AndreasBriese/bbloom 'Add' 65536 items (10 repetitions): 6595800 ns (100 ns/op)
|
||||||
|
github.com/AndreasBriese/bbloom 'Has' 65536 items (10 repetitions): 5986600 ns (91 ns/op)
|
||||||
|
github.com/AndreasBriese/bloom 'Add' 65536 items (10 repetitions): 6304684 ns (96 ns/op)
|
||||||
|
github.com/AndreasBriese/bloom 'Has' 65536 items (10 repetitions): 6568663 ns (100 ns/op)
|
||||||
|
|
||||||
|
github.com/willf/bloom 'Add' 65536 items (10 repetitions): 24367224 ns (371 ns/op)
|
||||||
|
github.com/willf/bloom 'Test' 65536 items (10 repetitions): 21881142 ns (333 ns/op)
|
||||||
|
github.com/dataence/bloom/standard 'Add' 65536 items (10 repetitions): 23041644 ns (351 ns/op)
|
||||||
|
github.com/dataence/bloom/standard 'Check' 65536 items (10 repetitions): 19153133 ns (292 ns/op)
|
||||||
|
github.com/cabello/bloom 'Add' 65536 items (10 repetitions): 131921507 ns (2012 ns/op)
|
||||||
|
github.com/cabello/bloom 'Contains' 65536 items (10 repetitions): 131108962 ns (2000 ns/op)
|
||||||
|
|
||||||
|
(on MBPro15 OSX10.8.5 i7 4Core 2.4Ghz)
|
||||||
|
|
||||||
|
|
||||||
|
With 32bit bloom filters (bloom32) using modified sdbm, bloom32 does hashing with only 2 bit shifts, one xor and one substraction per byte. smdb is about as fast as fnv64a but gives less collisions with the dataset (see mask above). bloom.New(float64(10 * 1<<16),float64(7)) populated with 1<<16 random items from the dataset (see above) and tested against the rest results in less than 0.05% collisions.
|
||||||
202
vendor/github.com/dgraph-io/ristretto/z/bbloom.go
generated
vendored
Normal file
202
vendor/github.com/dgraph-io/ristretto/z/bbloom.go
generated
vendored
Normal file
|
|
@ -0,0 +1,202 @@
|
||||||
|
// The MIT License (MIT)
|
||||||
|
// Copyright (c) 2014 Andreas Briese, eduToolbox@Bri-C GmbH, Sarstedt
|
||||||
|
|
||||||
|
// Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
// this software and associated documentation files (the "Software"), to deal in
|
||||||
|
// the Software without restriction, including without limitation the rights to
|
||||||
|
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||||
|
// the Software, and to permit persons to whom the Software is furnished to do so,
|
||||||
|
// subject to the following conditions:
|
||||||
|
|
||||||
|
// The above copyright notice and this permission notice shall be included in all
|
||||||
|
// copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||||
|
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||||
|
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||||
|
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||||
|
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
package z
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"log"
|
||||||
|
"math"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
// helper
|
||||||
|
var mask = []uint8{1, 2, 4, 8, 16, 32, 64, 128}
|
||||||
|
|
||||||
|
func getSize(ui64 uint64) (size uint64, exponent uint64) {
|
||||||
|
if ui64 < uint64(512) {
|
||||||
|
ui64 = uint64(512)
|
||||||
|
}
|
||||||
|
size = uint64(1)
|
||||||
|
for size < ui64 {
|
||||||
|
size <<= 1
|
||||||
|
exponent++
|
||||||
|
}
|
||||||
|
return size, exponent
|
||||||
|
}
|
||||||
|
|
||||||
|
func calcSizeByWrongPositives(numEntries, wrongs float64) (uint64, uint64) {
|
||||||
|
size := -1 * numEntries * math.Log(wrongs) / math.Pow(float64(0.69314718056), 2)
|
||||||
|
locs := math.Ceil(float64(0.69314718056) * size / numEntries)
|
||||||
|
return uint64(size), uint64(locs)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBloomFilter returns a new bloomfilter.
|
||||||
|
func NewBloomFilter(params ...float64) (bloomfilter *Bloom) {
|
||||||
|
var entries, locs uint64
|
||||||
|
if len(params) == 2 {
|
||||||
|
if params[1] < 1 {
|
||||||
|
entries, locs = calcSizeByWrongPositives(params[0], params[1])
|
||||||
|
} else {
|
||||||
|
entries, locs = uint64(params[0]), uint64(params[1])
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.Fatal("usage: New(float64(number_of_entries), float64(number_of_hashlocations))" +
|
||||||
|
" i.e. New(float64(1000), float64(3)) or New(float64(number_of_entries)," +
|
||||||
|
" float64(number_of_hashlocations)) i.e. New(float64(1000), float64(0.03))")
|
||||||
|
}
|
||||||
|
size, exponent := getSize(uint64(entries))
|
||||||
|
bloomfilter = &Bloom{
|
||||||
|
sizeExp: exponent,
|
||||||
|
size: size - 1,
|
||||||
|
setLocs: locs,
|
||||||
|
shift: 64 - exponent,
|
||||||
|
}
|
||||||
|
bloomfilter.Size(size)
|
||||||
|
return bloomfilter
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bloom filter
|
||||||
|
type Bloom struct {
|
||||||
|
bitset []uint64
|
||||||
|
ElemNum uint64
|
||||||
|
sizeExp uint64
|
||||||
|
size uint64
|
||||||
|
setLocs uint64
|
||||||
|
shift uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// <--- http://www.cse.yorku.ca/~oz/hash.html
|
||||||
|
// modified Berkeley DB Hash (32bit)
|
||||||
|
// hash is casted to l, h = 16bit fragments
|
||||||
|
// func (bl Bloom) absdbm(b *[]byte) (l, h uint64) {
|
||||||
|
// hash := uint64(len(*b))
|
||||||
|
// for _, c := range *b {
|
||||||
|
// hash = uint64(c) + (hash << 6) + (hash << bl.sizeExp) - hash
|
||||||
|
// }
|
||||||
|
// h = hash >> bl.shift
|
||||||
|
// l = hash << bl.shift >> bl.shift
|
||||||
|
// return l, h
|
||||||
|
// }
|
||||||
|
|
||||||
|
// Add adds hash of a key to the bloomfilter.
|
||||||
|
func (bl *Bloom) Add(hash uint64) {
|
||||||
|
h := hash >> bl.shift
|
||||||
|
l := hash << bl.shift >> bl.shift
|
||||||
|
for i := uint64(0); i < bl.setLocs; i++ {
|
||||||
|
bl.Set((h + i*l) & bl.size)
|
||||||
|
bl.ElemNum++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has checks if bit(s) for entry hash is/are set,
|
||||||
|
// returns true if the hash was added to the Bloom Filter.
|
||||||
|
func (bl Bloom) Has(hash uint64) bool {
|
||||||
|
h := hash >> bl.shift
|
||||||
|
l := hash << bl.shift >> bl.shift
|
||||||
|
for i := uint64(0); i < bl.setLocs; i++ {
|
||||||
|
switch bl.IsSet((h + i*l) & bl.size) {
|
||||||
|
case false:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddIfNotHas only Adds hash, if it's not present in the bloomfilter.
|
||||||
|
// Returns true if hash was added.
|
||||||
|
// Returns false if hash was already registered in the bloomfilter.
|
||||||
|
func (bl *Bloom) AddIfNotHas(hash uint64) bool {
|
||||||
|
if bl.Has(hash) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
bl.Add(hash)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size makes Bloom filter with as bitset of size sz.
|
||||||
|
func (bl *Bloom) Size(sz uint64) {
|
||||||
|
bl.bitset = make([]uint64, sz>>6)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear resets the Bloom filter.
|
||||||
|
func (bl *Bloom) Clear() {
|
||||||
|
for i := range bl.bitset {
|
||||||
|
bl.bitset[i] = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set sets the bit[idx] of bitset.
|
||||||
|
func (bl *Bloom) Set(idx uint64) {
|
||||||
|
ptr := unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[idx>>6])) + uintptr((idx%64)>>3))
|
||||||
|
*(*uint8)(ptr) |= mask[idx%8]
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsSet checks if bit[idx] of bitset is set, returns true/false.
|
||||||
|
func (bl *Bloom) IsSet(idx uint64) bool {
|
||||||
|
ptr := unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[idx>>6])) + uintptr((idx%64)>>3))
|
||||||
|
r := ((*(*uint8)(ptr)) >> (idx % 8)) & 1
|
||||||
|
return r == 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// bloomJSONImExport
|
||||||
|
// Im/Export structure used by JSONMarshal / JSONUnmarshal
|
||||||
|
type bloomJSONImExport struct {
|
||||||
|
FilterSet []byte
|
||||||
|
SetLocs uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewWithBoolset takes a []byte slice and number of locs per entry,
|
||||||
|
// returns the bloomfilter with a bitset populated according to the input []byte.
|
||||||
|
func newWithBoolset(bs *[]byte, locs uint64) *Bloom {
|
||||||
|
bloomfilter := NewBloomFilter(float64(len(*bs)<<3), float64(locs))
|
||||||
|
for i, b := range *bs {
|
||||||
|
*(*uint8)(unsafe.Pointer(uintptr(unsafe.Pointer(&bloomfilter.bitset[0])) + uintptr(i))) = b
|
||||||
|
}
|
||||||
|
return bloomfilter
|
||||||
|
}
|
||||||
|
|
||||||
|
// JSONUnmarshal takes JSON-Object (type bloomJSONImExport) as []bytes
|
||||||
|
// returns bloom32 / bloom64 object.
|
||||||
|
func JSONUnmarshal(dbData []byte) *Bloom {
|
||||||
|
bloomImEx := bloomJSONImExport{}
|
||||||
|
json.Unmarshal(dbData, &bloomImEx)
|
||||||
|
buf := bytes.NewBuffer(bloomImEx.FilterSet)
|
||||||
|
bs := buf.Bytes()
|
||||||
|
bf := newWithBoolset(&bs, bloomImEx.SetLocs)
|
||||||
|
return bf
|
||||||
|
}
|
||||||
|
|
||||||
|
// JSONMarshal returns JSON-object (type bloomJSONImExport) as []byte.
|
||||||
|
func (bl Bloom) JSONMarshal() []byte {
|
||||||
|
bloomImEx := bloomJSONImExport{}
|
||||||
|
bloomImEx.SetLocs = uint64(bl.setLocs)
|
||||||
|
bloomImEx.FilterSet = make([]byte, len(bl.bitset)<<3)
|
||||||
|
for i := range bloomImEx.FilterSet {
|
||||||
|
bloomImEx.FilterSet[i] = *(*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(&bl.bitset[0])) +
|
||||||
|
uintptr(i)))
|
||||||
|
}
|
||||||
|
data, err := json.Marshal(bloomImEx)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("json.Marshal failed: ", err)
|
||||||
|
}
|
||||||
|
return data
|
||||||
|
}
|
||||||
64
vendor/github.com/dgraph-io/ristretto/z/rtutil.go
generated
vendored
Normal file
64
vendor/github.com/dgraph-io/ristretto/z/rtutil.go
generated
vendored
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
// MIT License
|
||||||
|
|
||||||
|
// Copyright (c) 2019 Ewan Chou
|
||||||
|
|
||||||
|
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
// of this software and associated documentation files (the "Software"), to deal
|
||||||
|
// in the Software without restriction, including without limitation the rights
|
||||||
|
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
// copies of the Software, and to permit persons to whom the Software is
|
||||||
|
// furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
// The above copyright notice and this permission notice shall be included in all
|
||||||
|
// copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
// SOFTWARE.
|
||||||
|
|
||||||
|
package z
|
||||||
|
|
||||||
|
import (
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NanoTime returns the current time in nanoseconds from a monotonic clock.
|
||||||
|
//go:linkname NanoTime runtime.nanotime
|
||||||
|
func NanoTime() int64
|
||||||
|
|
||||||
|
// CPUTicks is a faster alternative to NanoTime to measure time duration.
|
||||||
|
//go:linkname CPUTicks runtime.cputicks
|
||||||
|
func CPUTicks() int64
|
||||||
|
|
||||||
|
type stringStruct struct {
|
||||||
|
str unsafe.Pointer
|
||||||
|
len int
|
||||||
|
}
|
||||||
|
|
||||||
|
//go:noescape
|
||||||
|
//go:linkname memhash runtime.memhash
|
||||||
|
func memhash(p unsafe.Pointer, h, s uintptr) uintptr
|
||||||
|
|
||||||
|
// MemHash is the hash function used by go map, it utilizes available hardware instructions(behaves
|
||||||
|
// as aeshash if aes instruction is available).
|
||||||
|
// NOTE: The hash seed changes for every process. So, this cannot be used as a persistent hash.
|
||||||
|
func MemHash(data []byte) uint64 {
|
||||||
|
ss := (*stringStruct)(unsafe.Pointer(&data))
|
||||||
|
return uint64(memhash(ss.str, 0, uintptr(ss.len)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// MemHashString is the hash function used by go map, it utilizes available hardware instructions
|
||||||
|
// (behaves as aeshash if aes instruction is available).
|
||||||
|
// NOTE: The hash seed changes for every process. So, this cannot be used as a persistent hash.
|
||||||
|
func MemHashString(str string) uint64 {
|
||||||
|
ss := (*stringStruct)(unsafe.Pointer(&str))
|
||||||
|
return uint64(memhash(ss.str, 0, uintptr(ss.len)))
|
||||||
|
}
|
||||||
|
|
||||||
|
// FastRand is a fast thread local random function.
|
||||||
|
//go:linkname FastRand runtime.fastrand
|
||||||
|
func FastRand() uint32
|
||||||
0
vendor/github.com/dgraph-io/ristretto/z/rtutil.s
generated
vendored
Normal file
0
vendor/github.com/dgraph-io/ristretto/z/rtutil.s
generated
vendored
Normal file
41
vendor/github.com/dgraph-io/ristretto/z/z.go
generated
vendored
Normal file
41
vendor/github.com/dgraph-io/ristretto/z/z.go
generated
vendored
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
/*
|
||||||
|
* Copyright 2019 Dgraph Labs, Inc. and Contributors
|
||||||
|
*
|
||||||
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
* you may not use this file except in compliance with the License.
|
||||||
|
* You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package z
|
||||||
|
|
||||||
|
// KeyToHash interprets the type of key and converts it to a uint64 hash.
|
||||||
|
func KeyToHash(key interface{}) uint64 {
|
||||||
|
switch k := key.(type) {
|
||||||
|
case uint64:
|
||||||
|
return k
|
||||||
|
case string:
|
||||||
|
return MemHashString(k)
|
||||||
|
case []byte:
|
||||||
|
return MemHash(k)
|
||||||
|
case byte:
|
||||||
|
return MemHash([]byte{k})
|
||||||
|
case int:
|
||||||
|
return uint64(k)
|
||||||
|
case int32:
|
||||||
|
return uint64(k)
|
||||||
|
case uint32:
|
||||||
|
return uint64(k)
|
||||||
|
case int64:
|
||||||
|
return uint64(k)
|
||||||
|
default:
|
||||||
|
panic("Key type not supported")
|
||||||
|
}
|
||||||
|
}
|
||||||
12
vendor/vendor.json
vendored
12
vendor/vendor.json
vendored
|
|
@ -74,6 +74,18 @@
|
||||||
"revision": "504e848d77ea4752b3057b8fb46da0e7f746ccf3",
|
"revision": "504e848d77ea4752b3057b8fb46da0e7f746ccf3",
|
||||||
"revisionTime": "2018-06-03T19:32:48Z"
|
"revisionTime": "2018-06-03T19:32:48Z"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "WBgnO6VroXxhpYS6kCiFS4jrW/c=",
|
||||||
|
"path": "github.com/dgraph-io/ristretto",
|
||||||
|
"revision": "ae326c3348e84b72d4f9db6b0ccb2f3551a0ed3e",
|
||||||
|
"revisionTime": "2019-09-24T01:22:51Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "fp+NtBZPTvGfWtsxZWUQct/e0ZM=",
|
||||||
|
"path": "github.com/dgraph-io/ristretto/z",
|
||||||
|
"revision": "ae326c3348e84b72d4f9db6b0ccb2f3551a0ed3e",
|
||||||
|
"revisionTime": "2019-09-24T01:22:51Z"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"checksumSHA1": "Ad8LPSCP9HctFrmskh+S5HpHXcs=",
|
"checksumSHA1": "Ad8LPSCP9HctFrmskh+S5HpHXcs=",
|
||||||
"path": "github.com/docker/docker/pkg/reexec",
|
"path": "github.com/docker/docker/pkg/reexec",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue