mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-20 10:52:25 +00:00
dumb pogreb impl
This commit is contained in:
Martin Holst Swende
parent
b4a4a4db71
commit
b6fdc7bcca
29 changed files with 2238 additions and 27 deletions
|
|
@ -286,29 +286,29 @@ func ImportPreimages(db ethdb.Database, fn string) error {
|
||||||
// ExportPreimages exports all known hash preimages into the specified file,
|
// ExportPreimages exports all known hash preimages into the specified file,
|
||||||
// truncating any data already present in the file.
|
// truncating any data already present in the file.
|
||||||
func ExportPreimages(db ethdb.Database, fn string) error {
|
func ExportPreimages(db ethdb.Database, fn string) error {
|
||||||
log.Info("Exporting preimages", "file", fn)
|
log.Info("Not exporting preimages", "file", fn)
|
||||||
|
|
||||||
// Open the file handle and potentially wrap with a gzip stream
|
//// Open the file handle and potentially wrap with a gzip stream
|
||||||
fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
|
//fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
|
||||||
if err != nil {
|
//if err != nil {
|
||||||
return err
|
// return err
|
||||||
}
|
//}
|
||||||
defer fh.Close()
|
//defer fh.Close()
|
||||||
|
//
|
||||||
var writer io.Writer = fh
|
//var writer io.Writer = fh
|
||||||
if strings.HasSuffix(fn, ".gz") {
|
//if strings.HasSuffix(fn, ".gz") {
|
||||||
writer = gzip.NewWriter(writer)
|
// writer = gzip.NewWriter(writer)
|
||||||
defer writer.(*gzip.Writer).Close()
|
// defer writer.(*gzip.Writer).Close()
|
||||||
}
|
//}
|
||||||
// Iterate over the preimages and export them
|
//// Iterate over the preimages and export them
|
||||||
it := db.NewIteratorWithPrefix([]byte("secure-key-"))
|
//it := db.NewIteratorWithPrefix([]byte("secure-key-"))
|
||||||
defer it.Release()
|
//defer it.Release()
|
||||||
|
//
|
||||||
for it.Next() {
|
//for it.Next() {
|
||||||
if err := rlp.Encode(writer, it.Value()); err != nil {
|
// if err := rlp.Encode(writer, it.Value()); err != nil {
|
||||||
return err
|
// return err
|
||||||
}
|
// }
|
||||||
}
|
//}
|
||||||
log.Info("Exported preimages", "file", fn)
|
//log.Info("Exported preimages", "file", fn)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,7 @@ import (
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/ethdb/leveldb"
|
"github.com/ethereum/go-ethereum/ethdb/pogreb"
|
||||||
"github.com/ethereum/go-ethereum/ethdb/memorydb"
|
"github.com/ethereum/go-ethereum/ethdb/memorydb"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
"github.com/olekukonko/tablewriter"
|
"github.com/olekukonko/tablewriter"
|
||||||
|
|
@ -197,7 +197,7 @@ func NewMemoryDatabaseWithCap(size int) ethdb.Database {
|
||||||
// NewLevelDBDatabase creates a persistent key-value database without a freezer
|
// NewLevelDBDatabase creates a persistent key-value database without a freezer
|
||||||
// moving immutable chain segments into cold storage.
|
// moving immutable chain segments into cold storage.
|
||||||
func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) {
|
func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) {
|
||||||
db, err := leveldb.New(file, cache, handles, namespace)
|
db, err := pogreb.New(file, cache)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -207,7 +207,8 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (
|
||||||
// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
|
// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
|
||||||
// freezer moving immutable chain segments into cold storage.
|
// freezer moving immutable chain segments into cold storage.
|
||||||
func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) {
|
func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) {
|
||||||
kvdb, err := leveldb.New(file, cache, handles, namespace)
|
//kvdb, err := leveldb.New(file, cache, handles, namespace)
|
||||||
|
kvdb, err := pogreb.New(file, cache)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -86,10 +86,16 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
|
||||||
)
|
)
|
||||||
// Ensure the datadir is not a symbolic link if it exists.
|
// Ensure the datadir is not a symbolic link if it exists.
|
||||||
if info, err := os.Lstat(datadir); !os.IsNotExist(err) {
|
if info, err := os.Lstat(datadir); !os.IsNotExist(err) {
|
||||||
if info.Mode()&os.ModeSymlink != 0 {
|
if info != nil {
|
||||||
|
fmt.Printf("Error %v \n", err)
|
||||||
|
}
|
||||||
|
if info != nil && info.Mode()&os.ModeSymlink != 0 {
|
||||||
log.Warn("Symbolic link ancient database is not supported", "path", datadir)
|
log.Warn("Symbolic link ancient database is not supported", "path", datadir)
|
||||||
return nil, errSymlinkDatadir
|
return nil, errSymlinkDatadir
|
||||||
}
|
}
|
||||||
|
//panic(err)
|
||||||
|
//return nil, err
|
||||||
|
|
||||||
}
|
}
|
||||||
// Leveldb uses LOCK as the filelock filename. To prevent the
|
// Leveldb uses LOCK as the filelock filename. To prevent the
|
||||||
// name collision, we use FLOCK as the lock name.
|
// name collision, we use FLOCK as the lock name.
|
||||||
|
|
|
||||||
17
vendor/github.com/akrylysov/pogreb/CHANGELOG.md
generated
vendored
Normal file
17
vendor/github.com/akrylysov/pogreb/CHANGELOG.md
generated
vendored
Normal file
|
|
@ -0,0 +1,17 @@
|
||||||
|
# Changelog
|
||||||
|
|
||||||
|
## [0.8.1] - 2019-06-30
|
||||||
|
### Fixed
|
||||||
|
- Fix panic when accessing closed database.
|
||||||
|
- Return error opening invalid database.
|
||||||
|
|
||||||
|
## [0.8] - 2019-03-30
|
||||||
|
### Changed
|
||||||
|
- ~2x write performance improvement on non-Windows.
|
||||||
|
|
||||||
|
## [0.7] - 2019-03-23
|
||||||
|
### Added
|
||||||
|
- Windows support (@mattn).
|
||||||
|
### Changed
|
||||||
|
- Improve freelist performance.
|
||||||
|
|
||||||
201
vendor/github.com/akrylysov/pogreb/LICENSE
generated
vendored
Normal file
201
vendor/github.com/akrylysov/pogreb/LICENSE
generated
vendored
Normal file
|
|
@ -0,0 +1,201 @@
|
||||||
|
Apache License
|
||||||
|
Version 2.0, January 2004
|
||||||
|
http://www.apache.org/licenses/
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||||
|
|
||||||
|
1. Definitions.
|
||||||
|
|
||||||
|
"License" shall mean the terms and conditions for use, reproduction,
|
||||||
|
and distribution as defined by Sections 1 through 9 of this document.
|
||||||
|
|
||||||
|
"Licensor" shall mean the copyright owner or entity authorized by
|
||||||
|
the copyright owner that is granting the License.
|
||||||
|
|
||||||
|
"Legal Entity" shall mean the union of the acting entity and all
|
||||||
|
other entities that control, are controlled by, or are under common
|
||||||
|
control with that entity. For the purposes of this definition,
|
||||||
|
"control" means (i) the power, direct or indirect, to cause the
|
||||||
|
direction or management of such entity, whether by contract or
|
||||||
|
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||||
|
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||||
|
|
||||||
|
"You" (or "Your") shall mean an individual or Legal Entity
|
||||||
|
exercising permissions granted by this License.
|
||||||
|
|
||||||
|
"Source" form shall mean the preferred form for making modifications,
|
||||||
|
including but not limited to software source code, documentation
|
||||||
|
source, and configuration files.
|
||||||
|
|
||||||
|
"Object" form shall mean any form resulting from mechanical
|
||||||
|
transformation or translation of a Source form, including but
|
||||||
|
not limited to compiled object code, generated documentation,
|
||||||
|
and conversions to other media types.
|
||||||
|
|
||||||
|
"Work" shall mean the work of authorship, whether in Source or
|
||||||
|
Object form, made available under the License, as indicated by a
|
||||||
|
copyright notice that is included in or attached to the work
|
||||||
|
(an example is provided in the Appendix below).
|
||||||
|
|
||||||
|
"Derivative Works" shall mean any work, whether in Source or Object
|
||||||
|
form, that is based on (or derived from) the Work and for which the
|
||||||
|
editorial revisions, annotations, elaborations, or other modifications
|
||||||
|
represent, as a whole, an original work of authorship. For the purposes
|
||||||
|
of this License, Derivative Works shall not include works that remain
|
||||||
|
separable from, or merely link (or bind by name) to the interfaces of,
|
||||||
|
the Work and Derivative Works thereof.
|
||||||
|
|
||||||
|
"Contribution" shall mean any work of authorship, including
|
||||||
|
the original version of the Work and any modifications or additions
|
||||||
|
to that Work or Derivative Works thereof, that is intentionally
|
||||||
|
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||||
|
or by an individual or Legal Entity authorized to submit on behalf of
|
||||||
|
the copyright owner. For the purposes of this definition, "submitted"
|
||||||
|
means any form of electronic, verbal, or written communication sent
|
||||||
|
to the Licensor or its representatives, including but not limited to
|
||||||
|
communication on electronic mailing lists, source code control systems,
|
||||||
|
and issue tracking systems that are managed by, or on behalf of, the
|
||||||
|
Licensor for the purpose of discussing and improving the Work, but
|
||||||
|
excluding communication that is conspicuously marked or otherwise
|
||||||
|
designated in writing by the copyright owner as "Not a Contribution."
|
||||||
|
|
||||||
|
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||||
|
on behalf of whom a Contribution has been received by Licensor and
|
||||||
|
subsequently incorporated within the Work.
|
||||||
|
|
||||||
|
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
copyright license to reproduce, prepare Derivative Works of,
|
||||||
|
publicly display, publicly perform, sublicense, and distribute the
|
||||||
|
Work and such Derivative Works in Source or Object form.
|
||||||
|
|
||||||
|
3. Grant of Patent License. Subject to the terms and conditions of
|
||||||
|
this License, each Contributor hereby grants to You a perpetual,
|
||||||
|
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||||
|
(except as stated in this section) patent license to make, have made,
|
||||||
|
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||||
|
where such license applies only to those patent claims licensable
|
||||||
|
by such Contributor that are necessarily infringed by their
|
||||||
|
Contribution(s) alone or by combination of their Contribution(s)
|
||||||
|
with the Work to which such Contribution(s) was submitted. If You
|
||||||
|
institute patent litigation against any entity (including a
|
||||||
|
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||||
|
or a Contribution incorporated within the Work constitutes direct
|
||||||
|
or contributory patent infringement, then any patent licenses
|
||||||
|
granted to You under this License for that Work shall terminate
|
||||||
|
as of the date such litigation is filed.
|
||||||
|
|
||||||
|
4. Redistribution. You may reproduce and distribute copies of the
|
||||||
|
Work or Derivative Works thereof in any medium, with or without
|
||||||
|
modifications, and in Source or Object form, provided that You
|
||||||
|
meet the following conditions:
|
||||||
|
|
||||||
|
(a) You must give any other recipients of the Work or
|
||||||
|
Derivative Works a copy of this License; and
|
||||||
|
|
||||||
|
(b) You must cause any modified files to carry prominent notices
|
||||||
|
stating that You changed the files; and
|
||||||
|
|
||||||
|
(c) You must retain, in the Source form of any Derivative Works
|
||||||
|
that You distribute, all copyright, patent, trademark, and
|
||||||
|
attribution notices from the Source form of the Work,
|
||||||
|
excluding those notices that do not pertain to any part of
|
||||||
|
the Derivative Works; and
|
||||||
|
|
||||||
|
(d) If the Work includes a "NOTICE" text file as part of its
|
||||||
|
distribution, then any Derivative Works that You distribute must
|
||||||
|
include a readable copy of the attribution notices contained
|
||||||
|
within such NOTICE file, excluding those notices that do not
|
||||||
|
pertain to any part of the Derivative Works, in at least one
|
||||||
|
of the following places: within a NOTICE text file distributed
|
||||||
|
as part of the Derivative Works; within the Source form or
|
||||||
|
documentation, if provided along with the Derivative Works; or,
|
||||||
|
within a display generated by the Derivative Works, if and
|
||||||
|
wherever such third-party notices normally appear. The contents
|
||||||
|
of the NOTICE file are for informational purposes only and
|
||||||
|
do not modify the License. You may add Your own attribution
|
||||||
|
notices within Derivative Works that You distribute, alongside
|
||||||
|
or as an addendum to the NOTICE text from the Work, provided
|
||||||
|
that such additional attribution notices cannot be construed
|
||||||
|
as modifying the License.
|
||||||
|
|
||||||
|
You may add Your own copyright statement to Your modifications and
|
||||||
|
may provide additional or different license terms and conditions
|
||||||
|
for use, reproduction, or distribution of Your modifications, or
|
||||||
|
for any such Derivative Works as a whole, provided Your use,
|
||||||
|
reproduction, and distribution of the Work otherwise complies with
|
||||||
|
the conditions stated in this License.
|
||||||
|
|
||||||
|
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||||
|
any Contribution intentionally submitted for inclusion in the Work
|
||||||
|
by You to the Licensor shall be under the terms and conditions of
|
||||||
|
this License, without any additional terms or conditions.
|
||||||
|
Notwithstanding the above, nothing herein shall supersede or modify
|
||||||
|
the terms of any separate license agreement you may have executed
|
||||||
|
with Licensor regarding such Contributions.
|
||||||
|
|
||||||
|
6. Trademarks. This License does not grant permission to use the trade
|
||||||
|
names, trademarks, service marks, or product names of the Licensor,
|
||||||
|
except as required for reasonable and customary use in describing the
|
||||||
|
origin of the Work and reproducing the content of the NOTICE file.
|
||||||
|
|
||||||
|
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||||
|
agreed to in writing, Licensor provides the Work (and each
|
||||||
|
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
implied, including, without limitation, any warranties or conditions
|
||||||
|
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||||
|
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||||
|
appropriateness of using or redistributing the Work and assume any
|
||||||
|
risks associated with Your exercise of permissions under this License.
|
||||||
|
|
||||||
|
8. Limitation of Liability. In no event and under no legal theory,
|
||||||
|
whether in tort (including negligence), contract, or otherwise,
|
||||||
|
unless required by applicable law (such as deliberate and grossly
|
||||||
|
negligent acts) or agreed to in writing, shall any Contributor be
|
||||||
|
liable to You for damages, including any direct, indirect, special,
|
||||||
|
incidental, or consequential damages of any character arising as a
|
||||||
|
result of this License or out of the use or inability to use the
|
||||||
|
Work (including but not limited to damages for loss of goodwill,
|
||||||
|
work stoppage, computer failure or malfunction, or any and all
|
||||||
|
other commercial damages or losses), even if such Contributor
|
||||||
|
has been advised of the possibility of such damages.
|
||||||
|
|
||||||
|
9. Accepting Warranty or Additional Liability. While redistributing
|
||||||
|
the Work or Derivative Works thereof, You may choose to offer,
|
||||||
|
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||||
|
or other liability obligations and/or rights consistent with this
|
||||||
|
License. However, in accepting such obligations, You may act only
|
||||||
|
on Your own behalf and on Your sole responsibility, not on behalf
|
||||||
|
of any other Contributor, and only if You agree to indemnify,
|
||||||
|
defend, and hold each Contributor harmless for any liability
|
||||||
|
incurred by, or claims asserted against, such Contributor by reason
|
||||||
|
of your accepting any such warranty or additional liability.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
APPENDIX: How to apply the Apache License to your work.
|
||||||
|
|
||||||
|
To apply the Apache License to your work, attach the following
|
||||||
|
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||||
|
replaced with your own identifying information. (Don't include
|
||||||
|
the brackets!) The text should be enclosed in the appropriate
|
||||||
|
comment syntax for the file format. We also recommend that a
|
||||||
|
file or class name and description of purpose be included on the
|
||||||
|
same "printed page" as the copyright notice for easier
|
||||||
|
identification within third-party archives.
|
||||||
|
|
||||||
|
Copyright [yyyy] [name of copyright owner]
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
98
vendor/github.com/akrylysov/pogreb/README.md
generated
vendored
Normal file
98
vendor/github.com/akrylysov/pogreb/README.md
generated
vendored
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
<p align="center"><img src="https://akrylysov.github.io/pogreb/logo.svg" width="300"></p>
|
||||||
|
|
||||||
|
# Pogreb [](https://godoc.org/github.com/akrylysov/pogreb) [](https://travis-ci.com/akrylysov/pogreb) [](https://ci.appveyor.com/project/akrylysov/pogreb/branch/master) [](https://goreportcard.com/report/github.com/akrylysov/pogreb)
|
||||||
|
|
||||||
|
Pogreb is an embedded key-value store for read-heavy workloads written in Go.
|
||||||
|
|
||||||
|
## Key characteristics
|
||||||
|
|
||||||
|
- 100% Go.
|
||||||
|
- Optimized for fast random lookups and infrequent bulk inserts.
|
||||||
|
- Can store larger-than-memory data sets.
|
||||||
|
- Low memory usage.
|
||||||
|
- All DB methods are safe for concurrent use by multiple goroutines.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
```sh
|
||||||
|
$ go get -u github.com/akrylysov/pogreb
|
||||||
|
```
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
### Opening a database
|
||||||
|
|
||||||
|
To open or create a new database, use the `pogreb.Open()` function:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"github.com/akrylysov/pogreb"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
db, err := pogreb.Open("pogreb.test", nil)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Writing to a database
|
||||||
|
|
||||||
|
Use the `DB.Put()` function to insert a new key/value pair:
|
||||||
|
|
||||||
|
```go
|
||||||
|
err := db.Put([]byte("testKey"), []byte("testValue"))
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Reading from a database
|
||||||
|
|
||||||
|
Use the `DB.Get()` function to retrieve the inserted value:
|
||||||
|
|
||||||
|
```go
|
||||||
|
val, err := db.Get([]byte("testKey"))
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
log.Printf("%s", val)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Iterating over items
|
||||||
|
|
||||||
|
Use the `DB.Items()` function which returns a new instance of `ItemIterator`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
it := db.Items()
|
||||||
|
for {
|
||||||
|
key, val, err := it.Next()
|
||||||
|
if err != nil {
|
||||||
|
if err != pogreb.ErrIterationDone {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
log.Printf("%s %s", key, val)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Performance
|
||||||
|
|
||||||
|
The benchmarking code can be found in the [pogreb-bench](https://github.com/akrylysov/pogreb-bench) repository.
|
||||||
|
|
||||||
|
Results of read performance benchmark of pogreb, goleveldb, bolt and badgerdb
|
||||||
|
on DigitalOcean 8 CPUs / 16 GB RAM / 160 GB SSD + Ubuntu 16.04.3 (higher is better):
|
||||||
|
|
||||||
|
<p align="center"><img src="https://akrylysov.github.io/pogreb/read-bench.png" width="609"></p>
|
||||||
|
|
||||||
|
## Internals
|
||||||
|
|
||||||
|
[Pogreb - how it works](https://artem.krylysov.com/blog/2018/03/24/pogreb-key-value-store/).
|
||||||
15
vendor/github.com/akrylysov/pogreb/appveyor.yml
generated
vendored
Normal file
15
vendor/github.com/akrylysov/pogreb/appveyor.yml
generated
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
version: "{build}"
|
||||||
|
|
||||||
|
clone_folder: c:\gopath\src\github.com\akrylysov/pogreb
|
||||||
|
|
||||||
|
environment:
|
||||||
|
GOPATH: c:\gopath
|
||||||
|
|
||||||
|
install:
|
||||||
|
- echo %PATH%
|
||||||
|
- echo %GOPATH%
|
||||||
|
- go version
|
||||||
|
- go env
|
||||||
|
|
||||||
|
build_script:
|
||||||
|
- go test -v ./...
|
||||||
121
vendor/github.com/akrylysov/pogreb/bucket.go
generated
vendored
Normal file
121
vendor/github.com/akrylysov/pogreb/bucket.go
generated
vendored
Normal file
|
|
@ -0,0 +1,121 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
|
||||||
|
"github.com/akrylysov/pogreb/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
type slot struct {
|
||||||
|
hash uint32
|
||||||
|
keySize uint16
|
||||||
|
valueSize uint32
|
||||||
|
kvOffset int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sl slot) kvSize() uint32 {
|
||||||
|
return uint32(sl.keySize) + sl.valueSize
|
||||||
|
}
|
||||||
|
|
||||||
|
type bucket struct {
|
||||||
|
slots [slotsPerBucket]slot
|
||||||
|
next int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type bucketHandle struct {
|
||||||
|
bucket
|
||||||
|
file fs.MmapFile
|
||||||
|
offset int64
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
bucketSize uint32 = 512
|
||||||
|
)
|
||||||
|
|
||||||
|
func align512(n uint32) uint32 {
|
||||||
|
return (n + 511) &^ 511
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b bucket) MarshalBinary() ([]byte, error) {
|
||||||
|
buf := make([]byte, bucketSize)
|
||||||
|
data := buf
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
sl := b.slots[i]
|
||||||
|
binary.LittleEndian.PutUint32(buf[:4], sl.hash)
|
||||||
|
binary.LittleEndian.PutUint16(buf[4:6], sl.keySize)
|
||||||
|
binary.LittleEndian.PutUint32(buf[6:10], sl.valueSize)
|
||||||
|
binary.LittleEndian.PutUint64(buf[10:18], uint64(sl.kvOffset))
|
||||||
|
buf = buf[18:]
|
||||||
|
}
|
||||||
|
binary.LittleEndian.PutUint64(buf[:8], uint64(b.next))
|
||||||
|
return data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *bucket) UnmarshalBinary(data []byte) error {
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
_ = data[18] // bounds check hint to compiler; see golang.org/issue/14808
|
||||||
|
b.slots[i].hash = binary.LittleEndian.Uint32(data[:4])
|
||||||
|
b.slots[i].keySize = binary.LittleEndian.Uint16(data[4:6])
|
||||||
|
b.slots[i].valueSize = binary.LittleEndian.Uint32(data[6:10])
|
||||||
|
b.slots[i].kvOffset = int64(binary.LittleEndian.Uint64(data[10:18]))
|
||||||
|
data = data[18:]
|
||||||
|
}
|
||||||
|
b.next = int64(binary.LittleEndian.Uint64(data[:8]))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *bucket) del(slotIdx int) {
|
||||||
|
i := slotIdx
|
||||||
|
for ; i < slotsPerBucket-1; i++ {
|
||||||
|
b.slots[i] = b.slots[i+1]
|
||||||
|
}
|
||||||
|
b.slots[i] = slot{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *bucketHandle) read() error {
|
||||||
|
buf, err := b.file.Slice(b.offset, b.offset+int64(bucketSize))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return b.UnmarshalBinary(buf)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *bucketHandle) write() error {
|
||||||
|
buf, err := b.MarshalBinary()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = b.file.WriteAt(buf, b.offset)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
type slotWriter struct {
|
||||||
|
bucket *bucketHandle
|
||||||
|
slotIdx int
|
||||||
|
prevBuckets []*bucketHandle
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sw *slotWriter) insert(sl slot, db *DB) error {
|
||||||
|
if sw.slotIdx == slotsPerBucket {
|
||||||
|
nextBucket, err := db.createOverflowBucket()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sw.bucket.next = nextBucket.offset
|
||||||
|
sw.prevBuckets = append(sw.prevBuckets, sw.bucket)
|
||||||
|
sw.bucket = nextBucket
|
||||||
|
sw.slotIdx = 0
|
||||||
|
}
|
||||||
|
sw.bucket.slots[sw.slotIdx] = sl
|
||||||
|
sw.slotIdx++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sw *slotWriter) write() error {
|
||||||
|
for i := len(sw.prevBuckets) - 1; i >= 0; i-- {
|
||||||
|
if err := sw.prevBuckets[i].write(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sw.bucket.write()
|
||||||
|
}
|
||||||
47
vendor/github.com/akrylysov/pogreb/datafile.go
generated
vendored
Normal file
47
vendor/github.com/akrylysov/pogreb/datafile.go
generated
vendored
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
type dataFile struct {
|
||||||
|
file
|
||||||
|
fl freelist
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *dataFile) readKeyValue(sl slot) ([]byte, []byte, error) {
|
||||||
|
keyValue, err := f.Slice(sl.kvOffset, sl.kvOffset+int64(sl.kvSize()))
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return keyValue[:sl.keySize], keyValue[sl.keySize:], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *dataFile) readKey(sl slot) ([]byte, error) {
|
||||||
|
return f.Slice(sl.kvOffset, sl.kvOffset+int64(sl.keySize))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *dataFile) allocate(size uint32) (int64, error) {
|
||||||
|
size = align512(size)
|
||||||
|
if off := f.fl.allocate(size); off > 0 {
|
||||||
|
return off, nil
|
||||||
|
}
|
||||||
|
return f.extend(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *dataFile) free(size uint32, off int64) {
|
||||||
|
size = align512(size)
|
||||||
|
f.fl.free(off, size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *dataFile) writeKeyValue(key []byte, value []byte) (int64, error) {
|
||||||
|
dataLen := align512(uint32(len(key) + len(value)))
|
||||||
|
data := make([]byte, dataLen)
|
||||||
|
copy(data, key)
|
||||||
|
copy(data[len(key):], value)
|
||||||
|
off := f.fl.allocate(dataLen)
|
||||||
|
if off != -1 {
|
||||||
|
if _, err := f.WriteAt(data, off); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return f.append(data)
|
||||||
|
}
|
||||||
|
return off, nil
|
||||||
|
}
|
||||||
568
vendor/github.com/akrylysov/pogreb/db.go
generated
vendored
Normal file
568
vendor/github.com/akrylysov/pogreb/db.go
generated
vendored
Normal file
|
|
@ -0,0 +1,568 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"math"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/akrylysov/pogreb/fs"
|
||||||
|
"github.com/akrylysov/pogreb/hash"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
slotsPerBucket = 28
|
||||||
|
loadFactor = 0.7
|
||||||
|
indexPostfix = ".index"
|
||||||
|
lockPostfix = ".lock"
|
||||||
|
version = 1 // file format version
|
||||||
|
|
||||||
|
// MaxKeyLength is the maximum size of a key in bytes.
|
||||||
|
MaxKeyLength = 1 << 16
|
||||||
|
|
||||||
|
// MaxValueLength is the maximum size of a value in bytes.
|
||||||
|
MaxValueLength = 1 << 30
|
||||||
|
|
||||||
|
// MaxKeys is the maximum numbers of keys in the DB.
|
||||||
|
MaxKeys = math.MaxUint32
|
||||||
|
)
|
||||||
|
|
||||||
|
type dbInfo struct {
|
||||||
|
level uint8
|
||||||
|
count uint32
|
||||||
|
nBuckets uint32
|
||||||
|
splitBucketIdx uint32
|
||||||
|
freelistOff int64
|
||||||
|
hashSeed uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
// DB represents the key-value storage.
|
||||||
|
// All DB methods are safe for concurrent use by multiple goroutines.
|
||||||
|
type DB struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
index file
|
||||||
|
data dataFile
|
||||||
|
lock fs.LockFile
|
||||||
|
metrics Metrics
|
||||||
|
cancelSyncer context.CancelFunc
|
||||||
|
syncWrites bool
|
||||||
|
dbInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open opens or creates a new DB.
|
||||||
|
func Open(path string, opts *Options) (*DB, error) {
|
||||||
|
opts = opts.copyWithDefaults()
|
||||||
|
fileFlag := os.O_CREATE | os.O_RDWR
|
||||||
|
fsys := opts.FileSystem
|
||||||
|
fileMode := os.FileMode(0666)
|
||||||
|
lock, needsRecovery, err := fsys.CreateLockFile(path+lockPostfix, fileMode)
|
||||||
|
if err != nil {
|
||||||
|
if err == os.ErrExist {
|
||||||
|
err = errLocked
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
index, err := openFile(fsys, path+indexPostfix, fileFlag, fileMode)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
data, err := openFile(fsys, path, fileFlag, fileMode)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
db := &DB{
|
||||||
|
index: index,
|
||||||
|
data: dataFile{file: data},
|
||||||
|
lock: lock,
|
||||||
|
metrics: newMetrics(),
|
||||||
|
dbInfo: dbInfo{
|
||||||
|
nBuckets: 1,
|
||||||
|
freelistOff: -1,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if index.size == 0 {
|
||||||
|
if data.size != 0 {
|
||||||
|
if err := index.Close(); err != nil {
|
||||||
|
logger.Print(err)
|
||||||
|
}
|
||||||
|
if err := data.Close(); err != nil {
|
||||||
|
logger.Print(err)
|
||||||
|
}
|
||||||
|
if err := lock.Unlock(); err != nil {
|
||||||
|
logger.Print(err)
|
||||||
|
}
|
||||||
|
// Data file exists, but index is missing.
|
||||||
|
return nil, errCorrupted
|
||||||
|
}
|
||||||
|
seed, err := hash.RandSeed()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
db.hashSeed = seed
|
||||||
|
if _, err = db.index.extend(headerSize + bucketSize); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, err = db.data.extend(headerSize); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := db.writeHeader(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := db.readHeader(!needsRecovery); err != nil {
|
||||||
|
if err := index.Close(); err != nil {
|
||||||
|
logger.Print(err)
|
||||||
|
}
|
||||||
|
if err := data.Close(); err != nil {
|
||||||
|
logger.Print(err)
|
||||||
|
}
|
||||||
|
if err := lock.Unlock(); err != nil {
|
||||||
|
logger.Print(err)
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if needsRecovery {
|
||||||
|
if err := db.recover(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if opts.BackgroundSyncInterval > 0 {
|
||||||
|
db.startSyncer(opts.BackgroundSyncInterval)
|
||||||
|
} else if opts.BackgroundSyncInterval == -1 {
|
||||||
|
db.syncWrites = true
|
||||||
|
}
|
||||||
|
return db, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func bucketOffset(idx uint32) int64 {
|
||||||
|
return int64(headerSize) + (int64(bucketSize) * int64(idx))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) startSyncer(interval time.Duration) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
db.cancelSyncer = cancel
|
||||||
|
go func() {
|
||||||
|
var lastModifications int64
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
modifications := db.metrics.Puts.Value() + db.metrics.Dels.Value()
|
||||||
|
if modifications != lastModifications {
|
||||||
|
if err := db.Sync(); err != nil {
|
||||||
|
logger.Printf("Error synchronizing databse: %v", err)
|
||||||
|
}
|
||||||
|
lastModifications = modifications
|
||||||
|
}
|
||||||
|
time.Sleep(interval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) forEachBucket(startBucketIdx uint32, cb func(bucketHandle) (bool, error)) error {
|
||||||
|
off := bucketOffset(startBucketIdx)
|
||||||
|
f := db.index.MmapFile
|
||||||
|
for {
|
||||||
|
b := bucketHandle{file: f, offset: off}
|
||||||
|
if err := b.read(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if stop, err := cb(b); stop || err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if b.next == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
off = b.next
|
||||||
|
f = db.data.MmapFile
|
||||||
|
db.metrics.BucketProbes.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) createOverflowBucket() (*bucketHandle, error) {
|
||||||
|
off, err := db.data.allocate(bucketSize)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &bucketHandle{file: db.data, offset: off}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) writeHeader() error {
|
||||||
|
db.data.fl.defrag()
|
||||||
|
freelistOff, err := db.data.fl.write(db.data.file)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
db.dbInfo.freelistOff = freelistOff
|
||||||
|
h := header{
|
||||||
|
signature: signature,
|
||||||
|
version: version,
|
||||||
|
dbInfo: db.dbInfo,
|
||||||
|
}
|
||||||
|
return db.index.writeMarshalableAt(h, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) readHeader(readFreeList bool) error {
|
||||||
|
h := &header{}
|
||||||
|
if err := db.index.readUnmarshalableAt(h, headerSize, 0); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !bytes.Equal(h.signature[:], signature[:]) {
|
||||||
|
return errCorrupted
|
||||||
|
}
|
||||||
|
db.dbInfo = h.dbInfo
|
||||||
|
if readFreeList {
|
||||||
|
if err := db.data.fl.read(db.data.file, db.dbInfo.freelistOff); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
db.dbInfo.freelistOff = -1
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close closes the DB.
|
||||||
|
func (db *DB) Close() error {
|
||||||
|
db.mu.Lock()
|
||||||
|
defer db.mu.Unlock()
|
||||||
|
if db.cancelSyncer != nil {
|
||||||
|
db.cancelSyncer()
|
||||||
|
}
|
||||||
|
if err := db.writeHeader(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.data.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.index.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.lock.Unlock(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) bucketIndex(hash uint32) uint32 {
|
||||||
|
idx := hash & ((1 << db.level) - 1)
|
||||||
|
if idx < db.splitBucketIdx {
|
||||||
|
return hash & ((1 << (db.level + 1)) - 1)
|
||||||
|
}
|
||||||
|
return idx
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) hash(data []byte) uint32 {
|
||||||
|
return hash.Sum32WithSeed(data, db.hashSeed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get returns the value for the given key stored in the DB or nil if the key doesn't exist.
|
||||||
|
func (db *DB) Get(key []byte) ([]byte, error) {
|
||||||
|
h := db.hash(key)
|
||||||
|
db.metrics.Gets.Add(1)
|
||||||
|
db.mu.RLock()
|
||||||
|
defer db.mu.RUnlock()
|
||||||
|
var retValue []byte
|
||||||
|
err := db.forEachBucket(db.bucketIndex(h), func(b bucketHandle) (bool, error) {
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
sl := b.slots[i]
|
||||||
|
if sl.kvOffset == 0 {
|
||||||
|
return b.next == 0, nil
|
||||||
|
} else if h == sl.hash && uint16(len(key)) == sl.keySize {
|
||||||
|
slKey, value, err := db.data.readKeyValue(sl)
|
||||||
|
if err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
if bytes.Equal(key, slKey) {
|
||||||
|
retValue = value
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
db.metrics.HashCollisions.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return retValue, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Has returns true if the DB contains the given key.
|
||||||
|
func (db *DB) Has(key []byte) (bool, error) {
|
||||||
|
h := db.hash(key)
|
||||||
|
found := false
|
||||||
|
db.mu.RLock()
|
||||||
|
defer db.mu.RUnlock()
|
||||||
|
err := db.forEachBucket(db.bucketIndex(h), func(b bucketHandle) (bool, error) {
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
sl := b.slots[i]
|
||||||
|
if sl.kvOffset == 0 {
|
||||||
|
return b.next == 0, nil
|
||||||
|
} else if h == sl.hash && uint16(len(key)) == sl.keySize {
|
||||||
|
slKey, err := db.data.readKey(sl)
|
||||||
|
if err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
if bytes.Equal(key, slKey) {
|
||||||
|
found = true
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return found, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Items returns a new ItemIterator.
|
||||||
|
func (db *DB) Items() *ItemIterator {
|
||||||
|
return &ItemIterator{db: db}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) sync() error {
|
||||||
|
if err := db.data.Sync(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.index.Sync(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync commits the contents of the database to the backing FileSystem; this is effectively a noop for an in-memory database. It must only be called while the database is opened.
|
||||||
|
func (db *DB) Sync() error {
|
||||||
|
db.mu.Lock()
|
||||||
|
defer db.mu.Unlock()
|
||||||
|
return db.sync()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Put sets the value for the given key. It updates the value for the existing key.
|
||||||
|
func (db *DB) Put(key []byte, value []byte) error {
|
||||||
|
switch {
|
||||||
|
case len(key) == 0:
|
||||||
|
return errKeyEmpty
|
||||||
|
case len(key) > MaxKeyLength:
|
||||||
|
return errKeyTooLarge
|
||||||
|
case len(value) > MaxValueLength:
|
||||||
|
return errValueTooLarge
|
||||||
|
}
|
||||||
|
h := db.hash(key)
|
||||||
|
db.metrics.Puts.Add(1)
|
||||||
|
db.mu.Lock()
|
||||||
|
defer db.mu.Unlock()
|
||||||
|
if err := db.put(h, key, value); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if float64(db.count)/float64(db.nBuckets*slotsPerBucket) > loadFactor {
|
||||||
|
if err := db.split(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if db.syncWrites {
|
||||||
|
return db.sync()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) put(hash uint32, key []byte, value []byte) error {
|
||||||
|
var b *bucketHandle
|
||||||
|
var originalB *bucketHandle
|
||||||
|
slotIdx := 0
|
||||||
|
err := db.forEachBucket(db.bucketIndex(hash), func(curb bucketHandle) (bool, error) {
|
||||||
|
b = &curb
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
sl := b.slots[i]
|
||||||
|
slotIdx = i
|
||||||
|
if sl.kvOffset == 0 {
|
||||||
|
// Found an empty slot.
|
||||||
|
return true, nil
|
||||||
|
} else if hash == sl.hash && uint16(len(key)) == sl.keySize {
|
||||||
|
// Key already exists.
|
||||||
|
if slKey, err := db.data.readKey(sl); bytes.Equal(key, slKey) || err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if b.next == 0 {
|
||||||
|
// Couldn't find free space in the current bucketHandle, creating a new overflow bucketHandle.
|
||||||
|
nextBucket, err := db.createOverflowBucket()
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
b.next = nextBucket.offset
|
||||||
|
originalB = b
|
||||||
|
b = nextBucket
|
||||||
|
slotIdx = 0
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inserting a new item.
|
||||||
|
if b.slots[slotIdx].kvOffset == 0 {
|
||||||
|
if db.count == MaxKeys {
|
||||||
|
return errFull
|
||||||
|
}
|
||||||
|
db.count++
|
||||||
|
} else {
|
||||||
|
defer db.data.free(b.slots[slotIdx].kvSize(), b.slots[slotIdx].kvOffset)
|
||||||
|
}
|
||||||
|
|
||||||
|
b.slots[slotIdx] = slot{
|
||||||
|
hash: hash,
|
||||||
|
keySize: uint16(len(key)),
|
||||||
|
valueSize: uint32(len(value)),
|
||||||
|
}
|
||||||
|
if b.slots[slotIdx].kvOffset, err = db.data.writeKeyValue(key, value); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := b.write(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if originalB != nil {
|
||||||
|
return originalB.write()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) split() error {
|
||||||
|
updatedBucketIdx := db.splitBucketIdx
|
||||||
|
updatedBucketOff := bucketOffset(updatedBucketIdx)
|
||||||
|
updatedBucket := slotWriter{
|
||||||
|
bucket: &bucketHandle{file: db.index, offset: updatedBucketOff},
|
||||||
|
}
|
||||||
|
|
||||||
|
newBucketOff, err := db.index.extend(bucketSize)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
newBucket := slotWriter{
|
||||||
|
bucket: &bucketHandle{file: db.index, offset: newBucketOff},
|
||||||
|
}
|
||||||
|
|
||||||
|
db.splitBucketIdx++
|
||||||
|
if db.splitBucketIdx == 1<<db.level {
|
||||||
|
db.level++
|
||||||
|
db.splitBucketIdx = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
var overflowBuckets []int64
|
||||||
|
if err := db.forEachBucket(updatedBucketIdx, func(curb bucketHandle) (bool, error) {
|
||||||
|
for j := 0; j < slotsPerBucket; j++ {
|
||||||
|
sl := curb.slots[j]
|
||||||
|
if sl.kvOffset == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if db.bucketIndex(sl.hash) == updatedBucketIdx {
|
||||||
|
if err := updatedBucket.insert(sl, db); err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := newBucket.insert(sl, db); err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if curb.next != 0 {
|
||||||
|
overflowBuckets = append(overflowBuckets, curb.next)
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, off := range overflowBuckets {
|
||||||
|
db.data.free(bucketSize, off)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := newBucket.write(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := updatedBucket.write(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
db.nBuckets++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete deletes the given key from the DB.
|
||||||
|
func (db *DB) Delete(key []byte) error {
|
||||||
|
h := db.hash(key)
|
||||||
|
db.metrics.Dels.Add(1)
|
||||||
|
db.mu.Lock()
|
||||||
|
defer db.mu.Unlock()
|
||||||
|
b := bucketHandle{}
|
||||||
|
slotIdx := -1
|
||||||
|
err := db.forEachBucket(db.bucketIndex(h), func(curb bucketHandle) (bool, error) {
|
||||||
|
b = curb
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
sl := b.slots[i]
|
||||||
|
if sl.kvOffset == 0 {
|
||||||
|
return b.next == 0, nil
|
||||||
|
} else if h == sl.hash && uint16(len(key)) == sl.keySize {
|
||||||
|
slKey, err := db.data.readKey(sl)
|
||||||
|
if err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
if bytes.Equal(key, slKey) {
|
||||||
|
slotIdx = i
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
})
|
||||||
|
if slotIdx == -1 || err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sl := b.slots[slotIdx]
|
||||||
|
b.del(slotIdx)
|
||||||
|
if err := b.write(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
db.data.free(sl.kvSize(), sl.kvOffset)
|
||||||
|
db.count--
|
||||||
|
if db.syncWrites {
|
||||||
|
return db.sync()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the number of items in the DB.
|
||||||
|
func (db *DB) Count() uint32 {
|
||||||
|
db.mu.RLock()
|
||||||
|
defer db.mu.RUnlock()
|
||||||
|
return db.count
|
||||||
|
}
|
||||||
|
|
||||||
|
// Metrics returns the DB metrics.
|
||||||
|
func (db *DB) Metrics() Metrics {
|
||||||
|
return db.metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileSize returns the total size of the disk storage used by the DB.
|
||||||
|
func (db *DB) FileSize() (int64, error) {
|
||||||
|
db.mu.RLock()
|
||||||
|
defer db.mu.RUnlock()
|
||||||
|
var err error
|
||||||
|
is, err := db.index.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
ds, err := db.data.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
return is.Size() + ds.Size(), nil
|
||||||
|
}
|
||||||
4
vendor/github.com/akrylysov/pogreb/doc.go
generated
vendored
Normal file
4
vendor/github.com/akrylysov/pogreb/doc.go
generated
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
/*
|
||||||
|
Package pogreb implements an embedded key-value store for read-heavy workloads.
|
||||||
|
*/
|
||||||
|
package pogreb
|
||||||
14
vendor/github.com/akrylysov/pogreb/errors.go
generated
vendored
Normal file
14
vendor/github.com/akrylysov/pogreb/errors.go
generated
vendored
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
errKeyEmpty = errors.New("key is empty")
|
||||||
|
errKeyTooLarge = errors.New("key is too large")
|
||||||
|
errValueTooLarge = errors.New("value is too large")
|
||||||
|
errFull = errors.New("database is full")
|
||||||
|
errCorrupted = errors.New("database is corrupted")
|
||||||
|
errLocked = errors.New("database is locked")
|
||||||
|
)
|
||||||
63
vendor/github.com/akrylysov/pogreb/file.go
generated
vendored
Normal file
63
vendor/github.com/akrylysov/pogreb/file.go
generated
vendored
Normal file
|
|
@ -0,0 +1,63 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/akrylysov/pogreb/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
type file struct {
|
||||||
|
fs.MmapFile
|
||||||
|
size int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func openFile(fsyst fs.FileSystem, name string, flag int, perm os.FileMode) (file, error) {
|
||||||
|
fi, err := fsyst.OpenFile(name, flag, perm)
|
||||||
|
f := file{}
|
||||||
|
if err != nil {
|
||||||
|
return f, err
|
||||||
|
}
|
||||||
|
f.MmapFile = fi
|
||||||
|
stat, err := fi.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return f, err
|
||||||
|
}
|
||||||
|
f.size = stat.Size()
|
||||||
|
return f, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *file) extend(size uint32) (int64, error) {
|
||||||
|
off := f.size
|
||||||
|
if err := f.Truncate(off + int64(size)); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
f.size += int64(size)
|
||||||
|
return off, f.Mmap(f.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *file) append(data []byte) (int64, error) {
|
||||||
|
off := f.size
|
||||||
|
if _, err := f.WriteAt(data, off); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
f.size += int64(len(data))
|
||||||
|
return off, f.Mmap(f.size)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *file) writeMarshalableAt(m encoding.BinaryMarshaler, off int64) error {
|
||||||
|
buf, err := m.MarshalBinary()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err = f.WriteAt(buf, off)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *file) readUnmarshalableAt(m encoding.BinaryUnmarshaler, size uint32, off int64) error {
|
||||||
|
buf := make([]byte, size)
|
||||||
|
if _, err := f.ReadAt(buf, off); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return m.UnmarshalBinary(buf)
|
||||||
|
}
|
||||||
149
vendor/github.com/akrylysov/pogreb/freelist.go
generated
vendored
Normal file
149
vendor/github.com/akrylysov/pogreb/freelist.go
generated
vendored
Normal file
|
|
@ -0,0 +1,149 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
type block struct {
|
||||||
|
offset int64
|
||||||
|
size uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
type freelist struct {
|
||||||
|
blocks []block
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fl *freelist) search(size uint32) int {
|
||||||
|
return sort.Search(len(fl.blocks), func(i int) bool {
|
||||||
|
return fl.blocks[i].size >= size
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fl *freelist) free(off int64, size uint32) {
|
||||||
|
if size == 0 {
|
||||||
|
panic("unable to free zero bytes")
|
||||||
|
}
|
||||||
|
i := fl.search(size)
|
||||||
|
if i < len(fl.blocks) && off == fl.blocks[i].offset {
|
||||||
|
panic("freeing already freed offset")
|
||||||
|
}
|
||||||
|
|
||||||
|
fl.blocks = append(fl.blocks, block{})
|
||||||
|
copy(fl.blocks[i+1:], fl.blocks[i:])
|
||||||
|
fl.blocks[i] = block{offset: off, size: size}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fl *freelist) allocate(size uint32) int64 {
|
||||||
|
if size == 0 {
|
||||||
|
panic("unable to allocate zero bytes")
|
||||||
|
}
|
||||||
|
i := fl.search(size)
|
||||||
|
if i >= len(fl.blocks) {
|
||||||
|
return -1
|
||||||
|
}
|
||||||
|
off := fl.blocks[i].offset
|
||||||
|
if fl.blocks[i].size == size {
|
||||||
|
copy(fl.blocks[i:], fl.blocks[i+1:])
|
||||||
|
fl.blocks[len(fl.blocks)-1] = block{}
|
||||||
|
fl.blocks = fl.blocks[:len(fl.blocks)-1]
|
||||||
|
} else {
|
||||||
|
fl.blocks[i].size -= size
|
||||||
|
fl.blocks[i].offset += int64(size)
|
||||||
|
}
|
||||||
|
return off
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fl *freelist) defrag() {
|
||||||
|
if len(fl.blocks) <= 1 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sort.Slice(fl.blocks, func(i, j int) bool {
|
||||||
|
return fl.blocks[i].offset < fl.blocks[j].offset
|
||||||
|
})
|
||||||
|
var merged []block
|
||||||
|
curOff := fl.blocks[0].offset
|
||||||
|
curSize := fl.blocks[0].size
|
||||||
|
for i := 1; i < len(fl.blocks); i++ {
|
||||||
|
if curOff+int64(curSize) == fl.blocks[i].offset {
|
||||||
|
curSize += fl.blocks[i].size
|
||||||
|
} else {
|
||||||
|
merged = append(merged, block{size: curSize, offset: curOff})
|
||||||
|
curOff = fl.blocks[i].offset
|
||||||
|
curSize = fl.blocks[i].size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
merged = append(merged, block{offset: curOff, size: curSize})
|
||||||
|
sort.Slice(merged, func(i, j int) bool {
|
||||||
|
return merged[i].size < merged[j].size
|
||||||
|
})
|
||||||
|
fl.blocks = merged
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fl *freelist) MarshalBinary() ([]byte, error) {
|
||||||
|
size := fl.binarySize()
|
||||||
|
buf := make([]byte, size)
|
||||||
|
data := buf
|
||||||
|
binary.LittleEndian.PutUint32(data[:4], uint32(len(fl.blocks)))
|
||||||
|
data = data[4:]
|
||||||
|
for i := 0; i < len(fl.blocks); i++ {
|
||||||
|
binary.LittleEndian.PutUint64(data[:8], uint64(fl.blocks[i].offset))
|
||||||
|
binary.LittleEndian.PutUint32(data[8:12], fl.blocks[i].size)
|
||||||
|
data = data[12:]
|
||||||
|
}
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fl *freelist) binarySize() uint32 {
|
||||||
|
return uint32(4 + (8+4)*len(fl.blocks)) // FIXME: this is ugly
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fl *freelist) read(f file, off int64) error {
|
||||||
|
if off == -1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
buf := make([]byte, 4)
|
||||||
|
if _, err := f.ReadAt(buf, off); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
n := binary.LittleEndian.Uint32(buf)
|
||||||
|
buf = make([]byte, (4+8)*n)
|
||||||
|
if _, err := f.ReadAt(buf, off+4); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for i := uint32(0); i < n; i++ {
|
||||||
|
blockOff := int64(binary.LittleEndian.Uint64(buf[:8]))
|
||||||
|
blockSize := binary.LittleEndian.Uint32(buf[8:12])
|
||||||
|
if blockOff != 0 {
|
||||||
|
fl.blocks = append(fl.blocks, block{size: blockSize, offset: blockOff})
|
||||||
|
}
|
||||||
|
buf = buf[12:]
|
||||||
|
}
|
||||||
|
fl.free(off, align512(4+(4+8)*n))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fl *freelist) write(f file) (int64, error) {
|
||||||
|
if len(fl.blocks) == 0 {
|
||||||
|
return -1, nil
|
||||||
|
}
|
||||||
|
marshaledSize := align512(fl.binarySize())
|
||||||
|
i := fl.search(marshaledSize)
|
||||||
|
var off int64
|
||||||
|
if i < len(fl.blocks) {
|
||||||
|
off = fl.blocks[i].offset
|
||||||
|
fl.blocks[i] = block{}
|
||||||
|
} else {
|
||||||
|
var err error
|
||||||
|
off, err = f.extend(marshaledSize)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
buf, err := fl.MarshalBinary()
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
_, err = f.WriteAt(buf, off)
|
||||||
|
return off, err
|
||||||
|
}
|
||||||
40
vendor/github.com/akrylysov/pogreb/fs/file.go
generated
vendored
Normal file
40
vendor/github.com/akrylysov/pogreb/fs/file.go
generated
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
||||||
|
package fs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
// File is the interface compatible with os.File.
|
||||||
|
type File interface {
|
||||||
|
io.Closer
|
||||||
|
io.Reader
|
||||||
|
io.ReaderAt
|
||||||
|
io.Seeker
|
||||||
|
io.Writer
|
||||||
|
io.WriterAt
|
||||||
|
|
||||||
|
Stat() (os.FileInfo, error)
|
||||||
|
Sync() error
|
||||||
|
Truncate(size int64) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// MmapFile represents a memory-mapped file.
|
||||||
|
type MmapFile interface {
|
||||||
|
File
|
||||||
|
Mmap(size int64) error
|
||||||
|
Slice(start int64, end int64) ([]byte, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LockFile represents a lock file.
|
||||||
|
type LockFile interface {
|
||||||
|
Unlock() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileSystem represents a virtual file system.
|
||||||
|
type FileSystem interface {
|
||||||
|
OpenFile(name string, flag int, perm os.FileMode) (MmapFile, error)
|
||||||
|
CreateLockFile(name string, perm os.FileMode) (LockFile, bool, error)
|
||||||
|
Stat(name string) (os.FileInfo, error)
|
||||||
|
Remove(name string) error
|
||||||
|
}
|
||||||
201
vendor/github.com/akrylysov/pogreb/fs/mem.go
generated
vendored
Normal file
201
vendor/github.com/akrylysov/pogreb/fs/mem.go
generated
vendored
Normal file
|
|
@ -0,0 +1,201 @@
|
||||||
|
package fs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type memfs struct {
|
||||||
|
files map[string]*memfile
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mem is a file system backed by memory.
|
||||||
|
var Mem = &memfs{files: map[string]*memfile{}}
|
||||||
|
|
||||||
|
func (fs *memfs) OpenFile(name string, flag int, perm os.FileMode) (MmapFile, error) {
|
||||||
|
// TODO: respect flag
|
||||||
|
f := fs.files[name]
|
||||||
|
if f == nil {
|
||||||
|
f = &memfile{}
|
||||||
|
fs.files[name] = f
|
||||||
|
} else if !f.closed {
|
||||||
|
return nil, os.ErrExist
|
||||||
|
} else {
|
||||||
|
f.closed = false
|
||||||
|
}
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *memfs) CreateLockFile(name string, perm os.FileMode) (LockFile, bool, error) {
|
||||||
|
f, err := fs.OpenFile(name, 0, perm)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return &memlockfile{f, name}, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *memfs) Stat(name string) (os.FileInfo, error) {
|
||||||
|
if f, ok := fs.files[name]; ok {
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
return nil, os.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *memfs) Remove(name string) error {
|
||||||
|
if _, ok := fs.files[name]; ok {
|
||||||
|
delete(fs.files, name)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return os.ErrNotExist
|
||||||
|
}
|
||||||
|
|
||||||
|
type memlockfile struct {
|
||||||
|
File
|
||||||
|
name string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *memlockfile) Unlock() error {
|
||||||
|
if err := f.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return Mem.Remove(f.name)
|
||||||
|
}
|
||||||
|
|
||||||
|
type memfile struct {
|
||||||
|
buf []byte
|
||||||
|
size int64
|
||||||
|
offset int64
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Close() error {
|
||||||
|
if m.closed {
|
||||||
|
return os.ErrClosed
|
||||||
|
}
|
||||||
|
m.closed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) ReadAt(p []byte, off int64) (int, error) {
|
||||||
|
if m.closed {
|
||||||
|
return 0, os.ErrClosed
|
||||||
|
}
|
||||||
|
n := len(p)
|
||||||
|
if int64(n) > m.size-off {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
copy(p, m.buf[off:off+int64(n)])
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Read(p []byte) (int, error) {
|
||||||
|
n, err := m.ReadAt(p, m.offset)
|
||||||
|
if err != nil {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
m.offset += int64(n)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) WriteAt(p []byte, off int64) (int, error) {
|
||||||
|
if m.closed {
|
||||||
|
return 0, os.ErrClosed
|
||||||
|
}
|
||||||
|
n := len(p)
|
||||||
|
if off == m.size {
|
||||||
|
m.buf = append(m.buf, p...)
|
||||||
|
m.size += int64(n)
|
||||||
|
} else if off+int64(n) > m.size {
|
||||||
|
panic("trying to write past EOF - undefined behavior")
|
||||||
|
} else {
|
||||||
|
copy(m.buf[off:off+int64(n)], p)
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Write(p []byte) (int, error) {
|
||||||
|
n, err := m.WriteAt(p, m.offset)
|
||||||
|
if err != nil {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
m.offset += int64(n)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Seek(offset int64, whence int) (int64, error) {
|
||||||
|
if m.closed {
|
||||||
|
return 0, os.ErrClosed
|
||||||
|
}
|
||||||
|
if whence == io.SeekEnd {
|
||||||
|
m.offset = m.size + offset
|
||||||
|
} else if whence == io.SeekStart {
|
||||||
|
m.offset = offset
|
||||||
|
} else if whence == io.SeekCurrent {
|
||||||
|
m.offset += offset
|
||||||
|
}
|
||||||
|
return m.offset, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Stat() (os.FileInfo, error) {
|
||||||
|
if m.closed {
|
||||||
|
return m, os.ErrClosed
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Sync() error {
|
||||||
|
if m.closed {
|
||||||
|
return os.ErrClosed
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Truncate(size int64) error {
|
||||||
|
if m.closed {
|
||||||
|
return os.ErrClosed
|
||||||
|
}
|
||||||
|
if size > m.size {
|
||||||
|
diff := int(size - m.size)
|
||||||
|
m.buf = append(m.buf, make([]byte, diff)...)
|
||||||
|
} else {
|
||||||
|
m.buf = m.buf[:m.size]
|
||||||
|
}
|
||||||
|
m.size = size
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Name() string {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Size() int64 {
|
||||||
|
return m.size
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Mode() os.FileMode {
|
||||||
|
return os.FileMode(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) ModTime() time.Time {
|
||||||
|
return time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) IsDir() bool {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Sys() interface{} {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Slice(start int64, end int64) ([]byte, error) {
|
||||||
|
if m.closed {
|
||||||
|
return nil, os.ErrClosed
|
||||||
|
}
|
||||||
|
return m.buf[start:end], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *memfile) Mmap(size int64) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
110
vendor/github.com/akrylysov/pogreb/fs/os.go
generated
vendored
Normal file
110
vendor/github.com/akrylysov/pogreb/fs/os.go
generated
vendored
Normal file
|
|
@ -0,0 +1,110 @@
|
||||||
|
package fs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
initialMmapSize = 1024 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
type osfile struct {
|
||||||
|
*os.File
|
||||||
|
data []byte
|
||||||
|
mmapSize int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type osfs struct{}
|
||||||
|
|
||||||
|
// OS is a file system backed by the os package.
|
||||||
|
var OS = &osfs{}
|
||||||
|
|
||||||
|
func (fs *osfs) OpenFile(name string, flag int, perm os.FileMode) (MmapFile, error) {
|
||||||
|
f, err := os.OpenFile(name, flag, perm)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
stat, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
mf := &osfile{f, nil, 0}
|
||||||
|
if stat.Size() > 0 {
|
||||||
|
if err := mf.Mmap(stat.Size()); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return mf, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *osfs) CreateLockFile(name string, perm os.FileMode) (LockFile, bool, error) {
|
||||||
|
return createLockFile(name, perm)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *osfs) Stat(name string) (os.FileInfo, error) {
|
||||||
|
return os.Stat(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (fs *osfs) Remove(name string) error {
|
||||||
|
return os.Remove(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
type oslockfile struct {
|
||||||
|
File
|
||||||
|
path string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *oslockfile) Unlock() error {
|
||||||
|
if err := os.Remove(f.path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return f.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *osfile) Slice(start int64, end int64) ([]byte, error) {
|
||||||
|
if f.data == nil {
|
||||||
|
return nil, os.ErrClosed
|
||||||
|
}
|
||||||
|
return f.data[start:end], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *osfile) Close() error {
|
||||||
|
if f.data != nil {
|
||||||
|
if err := munmap(f.data); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
f.data = nil
|
||||||
|
}
|
||||||
|
return f.File.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *osfile) Mmap(fileSize int64) error {
|
||||||
|
mmapSize := f.mmapSize
|
||||||
|
|
||||||
|
if mmapSize >= fileSize {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if mmapSize == 0 {
|
||||||
|
mmapSize = initialMmapSize
|
||||||
|
if mmapSize < fileSize {
|
||||||
|
mmapSize = fileSize
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := munmap(f.data); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
mmapSize *= 2
|
||||||
|
}
|
||||||
|
|
||||||
|
data, mappedSize, err := mmap(f.File, fileSize, mmapSize)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
madviceRandom(data)
|
||||||
|
|
||||||
|
f.data = data
|
||||||
|
f.mmapSize = mappedSize
|
||||||
|
return nil
|
||||||
|
}
|
||||||
44
vendor/github.com/akrylysov/pogreb/fs/os_unix.go
generated
vendored
Normal file
44
vendor/github.com/akrylysov/pogreb/fs/os_unix.go
generated
vendored
Normal file
|
|
@ -0,0 +1,44 @@
|
||||||
|
// +build !windows
|
||||||
|
|
||||||
|
package fs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
func mmap(f *os.File, fileSize int64, mmapSize int64) ([]byte, int64, error) {
|
||||||
|
p, err := syscall.Mmap(int(f.Fd()), 0, int(mmapSize), syscall.PROT_READ, syscall.MAP_SHARED)
|
||||||
|
return p, mmapSize, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func munmap(data []byte) error {
|
||||||
|
return syscall.Munmap(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
func madviceRandom(data []byte) error {
|
||||||
|
_, _, errno := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&data[0])), uintptr(len(data)), uintptr(syscall.MADV_RANDOM))
|
||||||
|
if errno != 0 {
|
||||||
|
return errno
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func createLockFile(name string, perm os.FileMode) (LockFile, bool, error) {
|
||||||
|
acquiredExisting := false
|
||||||
|
if _, err := os.Stat(name); err == nil {
|
||||||
|
acquiredExisting = true
|
||||||
|
}
|
||||||
|
f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, perm)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||||||
|
if err == syscall.EWOULDBLOCK {
|
||||||
|
err = os.ErrExist
|
||||||
|
}
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return &oslockfile{f, name}, acquiredExisting, nil
|
||||||
|
}
|
||||||
84
vendor/github.com/akrylysov/pogreb/fs/os_windows.go
generated
vendored
Normal file
84
vendor/github.com/akrylysov/pogreb/fs/os_windows.go
generated
vendored
Normal file
|
|
@ -0,0 +1,84 @@
|
||||||
|
// +build windows
|
||||||
|
|
||||||
|
package fs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
|
||||||
|
procLockFileEx = modkernel32.NewProc("LockFileEx")
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
errorLockViolation = 0x21
|
||||||
|
)
|
||||||
|
|
||||||
|
func mmap(f *os.File, fileSize int64, mmapSize int64) ([]byte, int64, error) {
|
||||||
|
size := fileSize
|
||||||
|
low, high := uint32(size), uint32(size>>32)
|
||||||
|
fmap, err := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, high, low, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
defer syscall.CloseHandle(fmap)
|
||||||
|
ptr, err := syscall.MapViewOfFile(fmap, syscall.FILE_MAP_READ, 0, 0, uintptr(size))
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
data := (*[1 << 30]byte)(unsafe.Pointer(ptr))[:size]
|
||||||
|
return data, size, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func munmap(data []byte) error {
|
||||||
|
return syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&data[0])))
|
||||||
|
}
|
||||||
|
|
||||||
|
func madviceRandom(data []byte) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func lockfile(f *os.File) error {
|
||||||
|
var ol syscall.Overlapped
|
||||||
|
|
||||||
|
r1, _, err := syscall.Syscall6(
|
||||||
|
procLockFileEx.Addr(),
|
||||||
|
6,
|
||||||
|
uintptr(f.Fd()), // handle
|
||||||
|
uintptr(0x0003),
|
||||||
|
uintptr(0), // reserved
|
||||||
|
uintptr(1), // locklow
|
||||||
|
uintptr(0), // lockhigh
|
||||||
|
uintptr(unsafe.Pointer(&ol)),
|
||||||
|
)
|
||||||
|
if r1 == 0 && (err == syscall.ERROR_FILE_EXISTS || err == errorLockViolation) {
|
||||||
|
return os.ErrExist
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func createLockFile(name string, perm os.FileMode) (LockFile, bool, error) {
|
||||||
|
acquiredExisting := false
|
||||||
|
if _, err := os.Stat(name); err == nil {
|
||||||
|
acquiredExisting = true
|
||||||
|
}
|
||||||
|
fd, err := syscall.CreateFile(&(syscall.StringToUTF16(name)[0]),
|
||||||
|
syscall.GENERIC_READ|syscall.GENERIC_WRITE,
|
||||||
|
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,
|
||||||
|
nil,
|
||||||
|
syscall.CREATE_ALWAYS,
|
||||||
|
syscall.FILE_ATTRIBUTE_NORMAL,
|
||||||
|
0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, os.ErrExist
|
||||||
|
}
|
||||||
|
f := os.NewFile(uintptr(fd), name)
|
||||||
|
if err := lockfile(f); err != nil {
|
||||||
|
f.Close()
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return &oslockfile{f, name}, acquiredExisting, nil
|
||||||
|
}
|
||||||
3
vendor/github.com/akrylysov/pogreb/go.mod
generated
vendored
Normal file
3
vendor/github.com/akrylysov/pogreb/go.mod
generated
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
||||||
|
module github.com/akrylysov/pogreb
|
||||||
|
|
||||||
|
go 1.12
|
||||||
61
vendor/github.com/akrylysov/pogreb/hash/murmurhash32.go
generated
vendored
Normal file
61
vendor/github.com/akrylysov/pogreb/hash/murmurhash32.go
generated
vendored
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
package hash
|
||||||
|
|
||||||
|
import (
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
c1 uint32 = 0xcc9e2d51
|
||||||
|
c2 uint32 = 0x1b873593
|
||||||
|
)
|
||||||
|
|
||||||
|
// Sum32WithSeed is a port of MurmurHash3_x86_32 function.
|
||||||
|
func Sum32WithSeed(data []byte, seed uint32) uint32 {
|
||||||
|
h1 := seed
|
||||||
|
|
||||||
|
nblocks := len(data) / 4
|
||||||
|
var p uintptr
|
||||||
|
if len(data) > 0 {
|
||||||
|
p = uintptr(unsafe.Pointer(&data[0]))
|
||||||
|
}
|
||||||
|
p1 := p + uintptr(4*nblocks)
|
||||||
|
for ; p < p1; p += 4 {
|
||||||
|
k1 := *(*uint32)(unsafe.Pointer(p))
|
||||||
|
|
||||||
|
k1 *= c1
|
||||||
|
k1 = (k1 << 15) | (k1 >> 17) // rotl32(k1, 15)
|
||||||
|
k1 *= c2
|
||||||
|
|
||||||
|
h1 ^= k1
|
||||||
|
h1 = (h1 << 13) | (h1 >> 19) // rotl32(h1, 13)
|
||||||
|
h1 = h1*5 + 0xe6546b64
|
||||||
|
}
|
||||||
|
|
||||||
|
tail := data[nblocks*4:]
|
||||||
|
|
||||||
|
var k1 uint32
|
||||||
|
switch len(tail) & 3 {
|
||||||
|
case 3:
|
||||||
|
k1 ^= uint32(tail[2]) << 16
|
||||||
|
fallthrough
|
||||||
|
case 2:
|
||||||
|
k1 ^= uint32(tail[1]) << 8
|
||||||
|
fallthrough
|
||||||
|
case 1:
|
||||||
|
k1 ^= uint32(tail[0])
|
||||||
|
k1 *= c1
|
||||||
|
k1 = (k1 << 15) | (k1 >> 17) // rotl32(k1, 15)
|
||||||
|
k1 *= c2
|
||||||
|
h1 ^= k1
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 ^= uint32(len(data))
|
||||||
|
|
||||||
|
h1 ^= h1 >> 16
|
||||||
|
h1 *= 0x85ebca6b
|
||||||
|
h1 ^= h1 >> 13
|
||||||
|
h1 *= 0xc2b2ae35
|
||||||
|
h1 ^= h1 >> 16
|
||||||
|
|
||||||
|
return h1
|
||||||
|
}
|
||||||
15
vendor/github.com/akrylysov/pogreb/hash/seed.go
generated
vendored
Normal file
15
vendor/github.com/akrylysov/pogreb/hash/seed.go
generated
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
package hash
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/binary"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RandSeed generates a random hash seed.
|
||||||
|
func RandSeed() (uint32, error) {
|
||||||
|
b := make([]byte, 4)
|
||||||
|
if _, err := rand.Read(b); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return binary.LittleEndian.Uint32(b), nil
|
||||||
|
}
|
||||||
46
vendor/github.com/akrylysov/pogreb/header.go
generated
vendored
Normal file
46
vendor/github.com/akrylysov/pogreb/header.go
generated
vendored
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
signature = [8]byte{'p', 'o', 'g', 'r', 'e', 'b', '\x0e', '\xfd'}
|
||||||
|
headerSize uint32
|
||||||
|
)
|
||||||
|
|
||||||
|
type header struct {
|
||||||
|
signature [8]byte
|
||||||
|
version uint32
|
||||||
|
dbInfo
|
||||||
|
_ [256]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
headerSize = align512(uint32(binary.Size(header{})))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h header) MarshalBinary() ([]byte, error) {
|
||||||
|
buf := make([]byte, headerSize)
|
||||||
|
copy(buf[:8], h.signature[:])
|
||||||
|
binary.LittleEndian.PutUint32(buf[8:12], h.version)
|
||||||
|
buf[12] = h.level
|
||||||
|
binary.LittleEndian.PutUint32(buf[13:17], h.count)
|
||||||
|
binary.LittleEndian.PutUint32(buf[17:21], h.nBuckets)
|
||||||
|
binary.LittleEndian.PutUint32(buf[21:25], h.splitBucketIdx)
|
||||||
|
binary.LittleEndian.PutUint64(buf[25:33], uint64(h.freelistOff))
|
||||||
|
binary.LittleEndian.PutUint32(buf[33:37], h.hashSeed)
|
||||||
|
return buf, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *header) UnmarshalBinary(data []byte) error {
|
||||||
|
copy(h.signature[:], data[:8])
|
||||||
|
h.version = binary.LittleEndian.Uint32(data[8:12])
|
||||||
|
h.level = data[12]
|
||||||
|
h.count = binary.LittleEndian.Uint32(data[13:17])
|
||||||
|
h.nBuckets = binary.LittleEndian.Uint32(data[17:21])
|
||||||
|
h.splitBucketIdx = binary.LittleEndian.Uint32(data[21:25])
|
||||||
|
h.freelistOff = int64(binary.LittleEndian.Uint64(data[25:33]))
|
||||||
|
h.hashSeed = binary.LittleEndian.Uint32(data[33:37])
|
||||||
|
return nil
|
||||||
|
}
|
||||||
65
vendor/github.com/akrylysov/pogreb/iterator.go
generated
vendored
Normal file
65
vendor/github.com/akrylysov/pogreb/iterator.go
generated
vendored
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrIterationDone is returned by ItemIterator.Next calls when there are no more items to return.
|
||||||
|
var ErrIterationDone = errors.New("no more items in iterator")
|
||||||
|
|
||||||
|
type item struct {
|
||||||
|
key []byte
|
||||||
|
value []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// ItemIterator is an iterator over DB key/value pairs. It iterates the items in an unspecified order.
|
||||||
|
type ItemIterator struct {
|
||||||
|
db *DB
|
||||||
|
nextBucketIdx uint32
|
||||||
|
queue []item
|
||||||
|
mu sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// Next returns the next key/value pair if available, otherwise it returns ErrIterationDone error.
|
||||||
|
func (it *ItemIterator) Next() ([]byte, []byte, error) {
|
||||||
|
it.mu.Lock()
|
||||||
|
defer it.mu.Unlock()
|
||||||
|
|
||||||
|
it.db.mu.RLock()
|
||||||
|
defer it.db.mu.RUnlock()
|
||||||
|
|
||||||
|
if len(it.queue) == 0 {
|
||||||
|
for it.nextBucketIdx < it.db.nBuckets {
|
||||||
|
err := it.db.forEachBucket(it.nextBucketIdx, func(b bucketHandle) (bool, error) {
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
sl := b.slots[i]
|
||||||
|
if sl.kvOffset == 0 {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
key, value, err := it.db.data.readKeyValue(sl)
|
||||||
|
if err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
it.queue = append(it.queue, item{key: key, value: value})
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
it.nextBucketIdx++
|
||||||
|
if len(it.queue) > 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(it.queue) > 0 {
|
||||||
|
item := it.queue[0]
|
||||||
|
it.queue = it.queue[1:]
|
||||||
|
return item.key, item.value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, nil, ErrIterationDone
|
||||||
|
}
|
||||||
15
vendor/github.com/akrylysov/pogreb/logger.go
generated
vendored
Normal file
15
vendor/github.com/akrylysov/pogreb/logger.go
generated
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
var logger = log.New(os.Stderr, "pogreb: ", 0)
|
||||||
|
|
||||||
|
// SetLogger sets the global logger.
|
||||||
|
func SetLogger(l *log.Logger) {
|
||||||
|
if l != nil {
|
||||||
|
logger = l
|
||||||
|
}
|
||||||
|
}
|
||||||
22
vendor/github.com/akrylysov/pogreb/metrics.go
generated
vendored
Normal file
22
vendor/github.com/akrylysov/pogreb/metrics.go
generated
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import "expvar"
|
||||||
|
|
||||||
|
// Metrics holds the DB metrics.
|
||||||
|
type Metrics struct {
|
||||||
|
Puts *expvar.Int
|
||||||
|
Dels *expvar.Int
|
||||||
|
Gets *expvar.Int
|
||||||
|
HashCollisions *expvar.Int
|
||||||
|
BucketProbes *expvar.Int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newMetrics() Metrics {
|
||||||
|
return Metrics{
|
||||||
|
Puts: &expvar.Int{},
|
||||||
|
Dels: &expvar.Int{},
|
||||||
|
Gets: &expvar.Int{},
|
||||||
|
HashCollisions: &expvar.Int{},
|
||||||
|
BucketProbes: &expvar.Int{},
|
||||||
|
}
|
||||||
|
}
|
||||||
29
vendor/github.com/akrylysov/pogreb/options.go
generated
vendored
Normal file
29
vendor/github.com/akrylysov/pogreb/options.go
generated
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/akrylysov/pogreb/fs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Options holds the optional DB parameters.
|
||||||
|
type Options struct {
|
||||||
|
// BackgroundSyncInterval sets the amount of time between background fsync() calls.
|
||||||
|
//
|
||||||
|
// Setting the value to 0 disables the automatic background synchronization.
|
||||||
|
// Setting the value to -1 makes the DB call fsync() after every write operation.
|
||||||
|
BackgroundSyncInterval time.Duration
|
||||||
|
|
||||||
|
FileSystem fs.FileSystem
|
||||||
|
}
|
||||||
|
|
||||||
|
func (src *Options) copyWithDefaults() *Options {
|
||||||
|
opts := Options{}
|
||||||
|
if src != nil {
|
||||||
|
opts = *src
|
||||||
|
}
|
||||||
|
if opts.FileSystem == nil {
|
||||||
|
opts.FileSystem = fs.OS
|
||||||
|
}
|
||||||
|
return &opts
|
||||||
|
}
|
||||||
154
vendor/github.com/akrylysov/pogreb/recovery.go
generated
vendored
Normal file
154
vendor/github.com/akrylysov/pogreb/recovery.go
generated
vendored
Normal file
|
|
@ -0,0 +1,154 @@
|
||||||
|
package pogreb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"sort"
|
||||||
|
)
|
||||||
|
|
||||||
|
func align51264(n int64) int64 {
|
||||||
|
return (n + 511) &^ 511
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncateFiles(db *DB) error {
|
||||||
|
db.index.size = align51264(db.index.size)
|
||||||
|
if err := db.index.Truncate(db.index.size); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.index.Mmap(db.index.size); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
db.data.size = align51264(db.data.size)
|
||||||
|
if err := db.data.Truncate(db.data.size); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := db.data.Mmap(db.data.size); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getUsedBlocks(db *DB) (uint32, []block, error) {
|
||||||
|
var itemCount uint32
|
||||||
|
var usedBlocks []block
|
||||||
|
for bucketIdx := uint32(0); bucketIdx < db.nBuckets; bucketIdx++ {
|
||||||
|
err := db.forEachBucket(bucketIdx, func(b bucketHandle) (bool, error) {
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
sl := b.slots[i]
|
||||||
|
if sl.kvOffset == 0 {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
itemCount++
|
||||||
|
usedBlocks = append(usedBlocks, block{size: align512(sl.kvSize()), offset: sl.kvOffset})
|
||||||
|
}
|
||||||
|
if b.next != 0 {
|
||||||
|
usedBlocks = append(usedBlocks, block{size: bucketSize, offset: b.next})
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return 0, nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return itemCount, usedBlocks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recoverSplitCrash(db *DB) error {
|
||||||
|
if db.nBuckets == 1 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
prevnBuckets := db.nBuckets - 1
|
||||||
|
prevLevel := uint8(math.Floor(math.Log2(float64(prevnBuckets))))
|
||||||
|
prevSplitBucketIdx := prevnBuckets - (uint32(1) << prevLevel)
|
||||||
|
splitCrash := false
|
||||||
|
err := db.forEachBucket(prevSplitBucketIdx, func(b bucketHandle) (bool, error) {
|
||||||
|
for i := 0; i < slotsPerBucket; i++ {
|
||||||
|
sl := b.slots[i]
|
||||||
|
if sl.kvOffset == 0 {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if db.bucketIndex(sl.hash) != prevSplitBucketIdx {
|
||||||
|
splitCrash = true
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !splitCrash {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
logger.Print("Detected split crash. Truncating index file...")
|
||||||
|
if err := db.index.Truncate(db.index.size - int64(bucketSize)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
db.index.size -= int64(bucketSize)
|
||||||
|
if err := db.index.Mmap(db.index.size); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
db.nBuckets = prevnBuckets
|
||||||
|
db.level = prevLevel
|
||||||
|
db.splitBucketIdx = prevSplitBucketIdx
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func recoverFreeList(db *DB, usedBlocks []block) error {
|
||||||
|
if len(usedBlocks) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
sort.Slice(usedBlocks, func(i, j int) bool {
|
||||||
|
return usedBlocks[i].offset < usedBlocks[j].offset
|
||||||
|
})
|
||||||
|
fl := freelist{}
|
||||||
|
expectedOff := int64(headerSize)
|
||||||
|
for _, bl := range usedBlocks {
|
||||||
|
if bl.offset > expectedOff {
|
||||||
|
fl.free(expectedOff, uint32(bl.offset-expectedOff))
|
||||||
|
}
|
||||||
|
expectedOff = bl.offset + int64(bl.size)
|
||||||
|
}
|
||||||
|
lastBlock := usedBlocks[len(usedBlocks)-1]
|
||||||
|
lastOffset := int64(lastBlock.size) + lastBlock.offset
|
||||||
|
if db.data.size > lastOffset {
|
||||||
|
fl.free(lastOffset, uint32(db.data.size-lastOffset))
|
||||||
|
logger.Println(lastBlock, db.data.size)
|
||||||
|
}
|
||||||
|
logger.Printf("Recovered freelist. Old len=%d; new len=%d\n", len(db.data.fl.blocks), len(fl.blocks))
|
||||||
|
db.data.fl = fl
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) recover() error {
|
||||||
|
logger.Println("Performing recovery...")
|
||||||
|
logger.Printf("Index file size=%d; data file size=%d\n", db.index.size, db.data.size)
|
||||||
|
logger.Printf("Header dbInfo %+v\n", db.dbInfo)
|
||||||
|
|
||||||
|
// Truncate index and data files.
|
||||||
|
if err := truncateFiles(db); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recover header.
|
||||||
|
db.nBuckets = uint32((db.index.size - int64(headerSize)) / int64(bucketSize))
|
||||||
|
db.level = uint8(math.Floor(math.Log2(float64(db.nBuckets))))
|
||||||
|
db.splitBucketIdx = db.nBuckets - (uint32(1) << db.level)
|
||||||
|
itemCount, usedBlocks, err := getUsedBlocks(db)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
db.count = itemCount
|
||||||
|
|
||||||
|
// Check if crash occurred during split.
|
||||||
|
if err := recoverSplitCrash(db); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
logger.Printf("Recovered dbInfo %+v\n", db.dbInfo)
|
||||||
|
|
||||||
|
// Recover free list.
|
||||||
|
if err := recoverFreeList(db, usedBlocks); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
logger.Println("Recovery complete.")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
18
vendor/vendor.json
vendored
18
vendor/vendor.json
vendored
|
|
@ -20,6 +20,24 @@
|
||||||
"revision": "5d049714c4a64225c3c79a7cf7d02f7fb5b96338",
|
"revision": "5d049714c4a64225c3c79a7cf7d02f7fb5b96338",
|
||||||
"revisionTime": "2018-01-16T20:38:02Z"
|
"revisionTime": "2018-01-16T20:38:02Z"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "EBi6awUmT1usWPTFQseEzp70578=",
|
||||||
|
"path": "github.com/akrylysov/pogreb",
|
||||||
|
"revision": "828d0ef698122ac786f0c769eeea8c2b41ac1493",
|
||||||
|
"revisionTime": "2019-06-30T21:56:05Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "7/INULWANAgDLXBI4eLIOgjjpzg=",
|
||||||
|
"path": "github.com/akrylysov/pogreb/fs",
|
||||||
|
"revision": "828d0ef698122ac786f0c769eeea8c2b41ac1493",
|
||||||
|
"revisionTime": "2019-06-30T21:56:05Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"checksumSHA1": "ykjZaRe9OBpXJFXqjlzInpzPM24=",
|
||||||
|
"path": "github.com/akrylysov/pogreb/hash",
|
||||||
|
"revision": "828d0ef698122ac786f0c769eeea8c2b41ac1493",
|
||||||
|
"revisionTime": "2019-06-30T21:56:05Z"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"checksumSHA1": "8skJYOdQytXjimcDPLRW4tonX3A=",
|
"checksumSHA1": "8skJYOdQytXjimcDPLRW4tonX3A=",
|
||||||
"path": "github.com/allegro/bigcache",
|
"path": "github.com/allegro/bigcache",
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue