mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-20 10:52:25 +00:00
dumb pogreb impl
This commit is contained in:
Martin Holst Swende
parent
b4a4a4db71
commit
b6fdc7bcca
29 changed files with 2238 additions and 27 deletions
|
|
@ -286,29 +286,29 @@ func ImportPreimages(db ethdb.Database, fn string) error {
|
|||
// ExportPreimages exports all known hash preimages into the specified file,
|
||||
// truncating any data already present in the file.
|
||||
func ExportPreimages(db ethdb.Database, fn string) error {
|
||||
log.Info("Exporting preimages", "file", fn)
|
||||
log.Info("Not exporting preimages", "file", fn)
|
||||
|
||||
// Open the file handle and potentially wrap with a gzip stream
|
||||
fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer fh.Close()
|
||||
|
||||
var writer io.Writer = fh
|
||||
if strings.HasSuffix(fn, ".gz") {
|
||||
writer = gzip.NewWriter(writer)
|
||||
defer writer.(*gzip.Writer).Close()
|
||||
}
|
||||
// Iterate over the preimages and export them
|
||||
it := db.NewIteratorWithPrefix([]byte("secure-key-"))
|
||||
defer it.Release()
|
||||
|
||||
for it.Next() {
|
||||
if err := rlp.Encode(writer, it.Value()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.Info("Exported preimages", "file", fn)
|
||||
//// Open the file handle and potentially wrap with a gzip stream
|
||||
//fh, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.ModePerm)
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
//defer fh.Close()
|
||||
//
|
||||
//var writer io.Writer = fh
|
||||
//if strings.HasSuffix(fn, ".gz") {
|
||||
// writer = gzip.NewWriter(writer)
|
||||
// defer writer.(*gzip.Writer).Close()
|
||||
//}
|
||||
//// Iterate over the preimages and export them
|
||||
//it := db.NewIteratorWithPrefix([]byte("secure-key-"))
|
||||
//defer it.Release()
|
||||
//
|
||||
//for it.Next() {
|
||||
// if err := rlp.Encode(writer, it.Value()); err != nil {
|
||||
// return err
|
||||
// }
|
||||
//}
|
||||
//log.Info("Exported preimages", "file", fn)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ import (
|
|||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/ethdb/leveldb"
|
||||
"github.com/ethereum/go-ethereum/ethdb/pogreb"
|
||||
"github.com/ethereum/go-ethereum/ethdb/memorydb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/olekukonko/tablewriter"
|
||||
|
|
@ -197,7 +197,7 @@ func NewMemoryDatabaseWithCap(size int) ethdb.Database {
|
|||
// NewLevelDBDatabase creates a persistent key-value database without a freezer
|
||||
// moving immutable chain segments into cold storage.
|
||||
func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (ethdb.Database, error) {
|
||||
db, err := leveldb.New(file, cache, handles, namespace)
|
||||
db, err := pogreb.New(file, cache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -207,7 +207,8 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string) (
|
|||
// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
|
||||
// freezer moving immutable chain segments into cold storage.
|
||||
func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) {
|
||||
kvdb, err := leveldb.New(file, cache, handles, namespace)
|
||||
//kvdb, err := leveldb.New(file, cache, handles, namespace)
|
||||
kvdb, err := pogreb.New(file, cache)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -86,10 +86,16 @@ func newFreezer(datadir string, namespace string) (*freezer, error) {
|
|||
)
|
||||
// Ensure the datadir is not a symbolic link if it exists.
|
||||
if info, err := os.Lstat(datadir); !os.IsNotExist(err) {
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
if info != nil {
|
||||
fmt.Printf("Error %v \n", err)
|
||||
}
|
||||
if info != nil && info.Mode()&os.ModeSymlink != 0 {
|
||||
log.Warn("Symbolic link ancient database is not supported", "path", datadir)
|
||||
return nil, errSymlinkDatadir
|
||||
}
|
||||
//panic(err)
|
||||
//return nil, err
|
||||
|
||||
}
|
||||
// Leveldb uses LOCK as the filelock filename. To prevent the
|
||||
// name collision, we use FLOCK as the lock name.
|
||||
|
|
|
|||
17
vendor/github.com/akrylysov/pogreb/CHANGELOG.md
generated
vendored
Normal file
17
vendor/github.com/akrylysov/pogreb/CHANGELOG.md
generated
vendored
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
# Changelog
|
||||
|
||||
## [0.8.1] - 2019-06-30
|
||||
### Fixed
|
||||
- Fix panic when accessing closed database.
|
||||
- Return error opening invalid database.
|
||||
|
||||
## [0.8] - 2019-03-30
|
||||
### Changed
|
||||
- ~2x write performance improvement on non-Windows.
|
||||
|
||||
## [0.7] - 2019-03-23
|
||||
### Added
|
||||
- Windows support (@mattn).
|
||||
### Changed
|
||||
- Improve freelist performance.
|
||||
|
||||
201
vendor/github.com/akrylysov/pogreb/LICENSE
generated
vendored
Normal file
201
vendor/github.com/akrylysov/pogreb/LICENSE
generated
vendored
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
98
vendor/github.com/akrylysov/pogreb/README.md
generated
vendored
Normal file
98
vendor/github.com/akrylysov/pogreb/README.md
generated
vendored
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
<p align="center"><img src="https://akrylysov.github.io/pogreb/logo.svg" width="300"></p>
|
||||
|
||||
# Pogreb [](https://godoc.org/github.com/akrylysov/pogreb) [](https://travis-ci.com/akrylysov/pogreb) [](https://ci.appveyor.com/project/akrylysov/pogreb/branch/master) [](https://goreportcard.com/report/github.com/akrylysov/pogreb)
|
||||
|
||||
Pogreb is an embedded key-value store for read-heavy workloads written in Go.
|
||||
|
||||
## Key characteristics
|
||||
|
||||
- 100% Go.
|
||||
- Optimized for fast random lookups and infrequent bulk inserts.
|
||||
- Can store larger-than-memory data sets.
|
||||
- Low memory usage.
|
||||
- All DB methods are safe for concurrent use by multiple goroutines.
|
||||
|
||||
## Installation
|
||||
|
||||
```sh
|
||||
$ go get -u github.com/akrylysov/pogreb
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
### Opening a database
|
||||
|
||||
To open or create a new database, use the `pogreb.Open()` function:
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/akrylysov/pogreb"
|
||||
)
|
||||
|
||||
func main() {
|
||||
db, err := pogreb.Open("pogreb.test", nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return
|
||||
}
|
||||
defer db.Close()
|
||||
}
|
||||
```
|
||||
|
||||
### Writing to a database
|
||||
|
||||
Use the `DB.Put()` function to insert a new key/value pair:
|
||||
|
||||
```go
|
||||
err := db.Put([]byte("testKey"), []byte("testValue"))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
```
|
||||
|
||||
### Reading from a database
|
||||
|
||||
Use the `DB.Get()` function to retrieve the inserted value:
|
||||
|
||||
```go
|
||||
val, err := db.Get([]byte("testKey"))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("%s", val)
|
||||
```
|
||||
|
||||
### Iterating over items
|
||||
|
||||
Use the `DB.Items()` function which returns a new instance of `ItemIterator`:
|
||||
|
||||
```go
|
||||
it := db.Items()
|
||||
for {
|
||||
key, val, err := it.Next()
|
||||
if err != nil {
|
||||
if err != pogreb.ErrIterationDone {
|
||||
log.Fatal(err)
|
||||
}
|
||||
break
|
||||
}
|
||||
log.Printf("%s %s", key, val)
|
||||
}
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
The benchmarking code can be found in the [pogreb-bench](https://github.com/akrylysov/pogreb-bench) repository.
|
||||
|
||||
Results of read performance benchmark of pogreb, goleveldb, bolt and badgerdb
|
||||
on DigitalOcean 8 CPUs / 16 GB RAM / 160 GB SSD + Ubuntu 16.04.3 (higher is better):
|
||||
|
||||
<p align="center"><img src="https://akrylysov.github.io/pogreb/read-bench.png" width="609"></p>
|
||||
|
||||
## Internals
|
||||
|
||||
[Pogreb - how it works](https://artem.krylysov.com/blog/2018/03/24/pogreb-key-value-store/).
|
||||
15
vendor/github.com/akrylysov/pogreb/appveyor.yml
generated
vendored
Normal file
15
vendor/github.com/akrylysov/pogreb/appveyor.yml
generated
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
version: "{build}"
|
||||
|
||||
clone_folder: c:\gopath\src\github.com\akrylysov/pogreb
|
||||
|
||||
environment:
|
||||
GOPATH: c:\gopath
|
||||
|
||||
install:
|
||||
- echo %PATH%
|
||||
- echo %GOPATH%
|
||||
- go version
|
||||
- go env
|
||||
|
||||
build_script:
|
||||
- go test -v ./...
|
||||
121
vendor/github.com/akrylysov/pogreb/bucket.go
generated
vendored
Normal file
121
vendor/github.com/akrylysov/pogreb/bucket.go
generated
vendored
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/akrylysov/pogreb/fs"
|
||||
)
|
||||
|
||||
type slot struct {
|
||||
hash uint32
|
||||
keySize uint16
|
||||
valueSize uint32
|
||||
kvOffset int64
|
||||
}
|
||||
|
||||
func (sl slot) kvSize() uint32 {
|
||||
return uint32(sl.keySize) + sl.valueSize
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
slots [slotsPerBucket]slot
|
||||
next int64
|
||||
}
|
||||
|
||||
type bucketHandle struct {
|
||||
bucket
|
||||
file fs.MmapFile
|
||||
offset int64
|
||||
}
|
||||
|
||||
const (
|
||||
bucketSize uint32 = 512
|
||||
)
|
||||
|
||||
func align512(n uint32) uint32 {
|
||||
return (n + 511) &^ 511
|
||||
}
|
||||
|
||||
func (b bucket) MarshalBinary() ([]byte, error) {
|
||||
buf := make([]byte, bucketSize)
|
||||
data := buf
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
sl := b.slots[i]
|
||||
binary.LittleEndian.PutUint32(buf[:4], sl.hash)
|
||||
binary.LittleEndian.PutUint16(buf[4:6], sl.keySize)
|
||||
binary.LittleEndian.PutUint32(buf[6:10], sl.valueSize)
|
||||
binary.LittleEndian.PutUint64(buf[10:18], uint64(sl.kvOffset))
|
||||
buf = buf[18:]
|
||||
}
|
||||
binary.LittleEndian.PutUint64(buf[:8], uint64(b.next))
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (b *bucket) UnmarshalBinary(data []byte) error {
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
_ = data[18] // bounds check hint to compiler; see golang.org/issue/14808
|
||||
b.slots[i].hash = binary.LittleEndian.Uint32(data[:4])
|
||||
b.slots[i].keySize = binary.LittleEndian.Uint16(data[4:6])
|
||||
b.slots[i].valueSize = binary.LittleEndian.Uint32(data[6:10])
|
||||
b.slots[i].kvOffset = int64(binary.LittleEndian.Uint64(data[10:18]))
|
||||
data = data[18:]
|
||||
}
|
||||
b.next = int64(binary.LittleEndian.Uint64(data[:8]))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *bucket) del(slotIdx int) {
|
||||
i := slotIdx
|
||||
for ; i < slotsPerBucket-1; i++ {
|
||||
b.slots[i] = b.slots[i+1]
|
||||
}
|
||||
b.slots[i] = slot{}
|
||||
}
|
||||
|
||||
func (b *bucketHandle) read() error {
|
||||
buf, err := b.file.Slice(b.offset, b.offset+int64(bucketSize))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return b.UnmarshalBinary(buf)
|
||||
}
|
||||
|
||||
func (b *bucketHandle) write() error {
|
||||
buf, err := b.MarshalBinary()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = b.file.WriteAt(buf, b.offset)
|
||||
return err
|
||||
}
|
||||
|
||||
type slotWriter struct {
|
||||
bucket *bucketHandle
|
||||
slotIdx int
|
||||
prevBuckets []*bucketHandle
|
||||
}
|
||||
|
||||
func (sw *slotWriter) insert(sl slot, db *DB) error {
|
||||
if sw.slotIdx == slotsPerBucket {
|
||||
nextBucket, err := db.createOverflowBucket()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sw.bucket.next = nextBucket.offset
|
||||
sw.prevBuckets = append(sw.prevBuckets, sw.bucket)
|
||||
sw.bucket = nextBucket
|
||||
sw.slotIdx = 0
|
||||
}
|
||||
sw.bucket.slots[sw.slotIdx] = sl
|
||||
sw.slotIdx++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sw *slotWriter) write() error {
|
||||
for i := len(sw.prevBuckets) - 1; i >= 0; i-- {
|
||||
if err := sw.prevBuckets[i].write(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return sw.bucket.write()
|
||||
}
|
||||
47
vendor/github.com/akrylysov/pogreb/datafile.go
generated
vendored
Normal file
47
vendor/github.com/akrylysov/pogreb/datafile.go
generated
vendored
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
package pogreb
|
||||
|
||||
type dataFile struct {
|
||||
file
|
||||
fl freelist
|
||||
}
|
||||
|
||||
func (f *dataFile) readKeyValue(sl slot) ([]byte, []byte, error) {
|
||||
keyValue, err := f.Slice(sl.kvOffset, sl.kvOffset+int64(sl.kvSize()))
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return keyValue[:sl.keySize], keyValue[sl.keySize:], nil
|
||||
}
|
||||
|
||||
func (f *dataFile) readKey(sl slot) ([]byte, error) {
|
||||
return f.Slice(sl.kvOffset, sl.kvOffset+int64(sl.keySize))
|
||||
}
|
||||
|
||||
func (f *dataFile) allocate(size uint32) (int64, error) {
|
||||
size = align512(size)
|
||||
if off := f.fl.allocate(size); off > 0 {
|
||||
return off, nil
|
||||
}
|
||||
return f.extend(size)
|
||||
}
|
||||
|
||||
func (f *dataFile) free(size uint32, off int64) {
|
||||
size = align512(size)
|
||||
f.fl.free(off, size)
|
||||
}
|
||||
|
||||
func (f *dataFile) writeKeyValue(key []byte, value []byte) (int64, error) {
|
||||
dataLen := align512(uint32(len(key) + len(value)))
|
||||
data := make([]byte, dataLen)
|
||||
copy(data, key)
|
||||
copy(data[len(key):], value)
|
||||
off := f.fl.allocate(dataLen)
|
||||
if off != -1 {
|
||||
if _, err := f.WriteAt(data, off); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
return f.append(data)
|
||||
}
|
||||
return off, nil
|
||||
}
|
||||
568
vendor/github.com/akrylysov/pogreb/db.go
generated
vendored
Normal file
568
vendor/github.com/akrylysov/pogreb/db.go
generated
vendored
Normal file
|
|
@ -0,0 +1,568 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"math"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/akrylysov/pogreb/fs"
|
||||
"github.com/akrylysov/pogreb/hash"
|
||||
)
|
||||
|
||||
const (
|
||||
slotsPerBucket = 28
|
||||
loadFactor = 0.7
|
||||
indexPostfix = ".index"
|
||||
lockPostfix = ".lock"
|
||||
version = 1 // file format version
|
||||
|
||||
// MaxKeyLength is the maximum size of a key in bytes.
|
||||
MaxKeyLength = 1 << 16
|
||||
|
||||
// MaxValueLength is the maximum size of a value in bytes.
|
||||
MaxValueLength = 1 << 30
|
||||
|
||||
// MaxKeys is the maximum numbers of keys in the DB.
|
||||
MaxKeys = math.MaxUint32
|
||||
)
|
||||
|
||||
type dbInfo struct {
|
||||
level uint8
|
||||
count uint32
|
||||
nBuckets uint32
|
||||
splitBucketIdx uint32
|
||||
freelistOff int64
|
||||
hashSeed uint32
|
||||
}
|
||||
|
||||
// DB represents the key-value storage.
|
||||
// All DB methods are safe for concurrent use by multiple goroutines.
|
||||
type DB struct {
|
||||
mu sync.RWMutex
|
||||
index file
|
||||
data dataFile
|
||||
lock fs.LockFile
|
||||
metrics Metrics
|
||||
cancelSyncer context.CancelFunc
|
||||
syncWrites bool
|
||||
dbInfo
|
||||
}
|
||||
|
||||
// Open opens or creates a new DB.
|
||||
func Open(path string, opts *Options) (*DB, error) {
|
||||
opts = opts.copyWithDefaults()
|
||||
fileFlag := os.O_CREATE | os.O_RDWR
|
||||
fsys := opts.FileSystem
|
||||
fileMode := os.FileMode(0666)
|
||||
lock, needsRecovery, err := fsys.CreateLockFile(path+lockPostfix, fileMode)
|
||||
if err != nil {
|
||||
if err == os.ErrExist {
|
||||
err = errLocked
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
index, err := openFile(fsys, path+indexPostfix, fileFlag, fileMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data, err := openFile(fsys, path, fileFlag, fileMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db := &DB{
|
||||
index: index,
|
||||
data: dataFile{file: data},
|
||||
lock: lock,
|
||||
metrics: newMetrics(),
|
||||
dbInfo: dbInfo{
|
||||
nBuckets: 1,
|
||||
freelistOff: -1,
|
||||
},
|
||||
}
|
||||
if index.size == 0 {
|
||||
if data.size != 0 {
|
||||
if err := index.Close(); err != nil {
|
||||
logger.Print(err)
|
||||
}
|
||||
if err := data.Close(); err != nil {
|
||||
logger.Print(err)
|
||||
}
|
||||
if err := lock.Unlock(); err != nil {
|
||||
logger.Print(err)
|
||||
}
|
||||
// Data file exists, but index is missing.
|
||||
return nil, errCorrupted
|
||||
}
|
||||
seed, err := hash.RandSeed()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db.hashSeed = seed
|
||||
if _, err = db.index.extend(headerSize + bucketSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = db.data.extend(headerSize); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := db.writeHeader(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err := db.readHeader(!needsRecovery); err != nil {
|
||||
if err := index.Close(); err != nil {
|
||||
logger.Print(err)
|
||||
}
|
||||
if err := data.Close(); err != nil {
|
||||
logger.Print(err)
|
||||
}
|
||||
if err := lock.Unlock(); err != nil {
|
||||
logger.Print(err)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if needsRecovery {
|
||||
if err := db.recover(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if opts.BackgroundSyncInterval > 0 {
|
||||
db.startSyncer(opts.BackgroundSyncInterval)
|
||||
} else if opts.BackgroundSyncInterval == -1 {
|
||||
db.syncWrites = true
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func bucketOffset(idx uint32) int64 {
|
||||
return int64(headerSize) + (int64(bucketSize) * int64(idx))
|
||||
}
|
||||
|
||||
func (db *DB) startSyncer(interval time.Duration) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
db.cancelSyncer = cancel
|
||||
go func() {
|
||||
var lastModifications int64
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
modifications := db.metrics.Puts.Value() + db.metrics.Dels.Value()
|
||||
if modifications != lastModifications {
|
||||
if err := db.Sync(); err != nil {
|
||||
logger.Printf("Error synchronizing databse: %v", err)
|
||||
}
|
||||
lastModifications = modifications
|
||||
}
|
||||
time.Sleep(interval)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (db *DB) forEachBucket(startBucketIdx uint32, cb func(bucketHandle) (bool, error)) error {
|
||||
off := bucketOffset(startBucketIdx)
|
||||
f := db.index.MmapFile
|
||||
for {
|
||||
b := bucketHandle{file: f, offset: off}
|
||||
if err := b.read(); err != nil {
|
||||
return err
|
||||
}
|
||||
if stop, err := cb(b); stop || err != nil {
|
||||
return err
|
||||
}
|
||||
if b.next == 0 {
|
||||
return nil
|
||||
}
|
||||
off = b.next
|
||||
f = db.data.MmapFile
|
||||
db.metrics.BucketProbes.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
func (db *DB) createOverflowBucket() (*bucketHandle, error) {
|
||||
off, err := db.data.allocate(bucketSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &bucketHandle{file: db.data, offset: off}, nil
|
||||
}
|
||||
|
||||
func (db *DB) writeHeader() error {
|
||||
db.data.fl.defrag()
|
||||
freelistOff, err := db.data.fl.write(db.data.file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db.dbInfo.freelistOff = freelistOff
|
||||
h := header{
|
||||
signature: signature,
|
||||
version: version,
|
||||
dbInfo: db.dbInfo,
|
||||
}
|
||||
return db.index.writeMarshalableAt(h, 0)
|
||||
}
|
||||
|
||||
func (db *DB) readHeader(readFreeList bool) error {
|
||||
h := &header{}
|
||||
if err := db.index.readUnmarshalableAt(h, headerSize, 0); err != nil {
|
||||
return err
|
||||
}
|
||||
if !bytes.Equal(h.signature[:], signature[:]) {
|
||||
return errCorrupted
|
||||
}
|
||||
db.dbInfo = h.dbInfo
|
||||
if readFreeList {
|
||||
if err := db.data.fl.read(db.data.file, db.dbInfo.freelistOff); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
db.dbInfo.freelistOff = -1
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the DB.
|
||||
func (db *DB) Close() error {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
if db.cancelSyncer != nil {
|
||||
db.cancelSyncer()
|
||||
}
|
||||
if err := db.writeHeader(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.data.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.index.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.lock.Unlock(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) bucketIndex(hash uint32) uint32 {
|
||||
idx := hash & ((1 << db.level) - 1)
|
||||
if idx < db.splitBucketIdx {
|
||||
return hash & ((1 << (db.level + 1)) - 1)
|
||||
}
|
||||
return idx
|
||||
}
|
||||
|
||||
func (db *DB) hash(data []byte) uint32 {
|
||||
return hash.Sum32WithSeed(data, db.hashSeed)
|
||||
}
|
||||
|
||||
// Get returns the value for the given key stored in the DB or nil if the key doesn't exist.
|
||||
func (db *DB) Get(key []byte) ([]byte, error) {
|
||||
h := db.hash(key)
|
||||
db.metrics.Gets.Add(1)
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
var retValue []byte
|
||||
err := db.forEachBucket(db.bucketIndex(h), func(b bucketHandle) (bool, error) {
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
sl := b.slots[i]
|
||||
if sl.kvOffset == 0 {
|
||||
return b.next == 0, nil
|
||||
} else if h == sl.hash && uint16(len(key)) == sl.keySize {
|
||||
slKey, value, err := db.data.readKeyValue(sl)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
if bytes.Equal(key, slKey) {
|
||||
retValue = value
|
||||
return true, nil
|
||||
}
|
||||
db.metrics.HashCollisions.Add(1)
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return retValue, nil
|
||||
}
|
||||
|
||||
// Has returns true if the DB contains the given key.
|
||||
func (db *DB) Has(key []byte) (bool, error) {
|
||||
h := db.hash(key)
|
||||
found := false
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
err := db.forEachBucket(db.bucketIndex(h), func(b bucketHandle) (bool, error) {
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
sl := b.slots[i]
|
||||
if sl.kvOffset == 0 {
|
||||
return b.next == 0, nil
|
||||
} else if h == sl.hash && uint16(len(key)) == sl.keySize {
|
||||
slKey, err := db.data.readKey(sl)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
if bytes.Equal(key, slKey) {
|
||||
found = true
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return found, nil
|
||||
}
|
||||
|
||||
// Items returns a new ItemIterator.
|
||||
func (db *DB) Items() *ItemIterator {
|
||||
return &ItemIterator{db: db}
|
||||
}
|
||||
|
||||
func (db *DB) sync() error {
|
||||
if err := db.data.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.index.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sync commits the contents of the database to the backing FileSystem; this is effectively a noop for an in-memory database. It must only be called while the database is opened.
|
||||
func (db *DB) Sync() error {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
return db.sync()
|
||||
}
|
||||
|
||||
// Put sets the value for the given key. It updates the value for the existing key.
|
||||
func (db *DB) Put(key []byte, value []byte) error {
|
||||
switch {
|
||||
case len(key) == 0:
|
||||
return errKeyEmpty
|
||||
case len(key) > MaxKeyLength:
|
||||
return errKeyTooLarge
|
||||
case len(value) > MaxValueLength:
|
||||
return errValueTooLarge
|
||||
}
|
||||
h := db.hash(key)
|
||||
db.metrics.Puts.Add(1)
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
if err := db.put(h, key, value); err != nil {
|
||||
return err
|
||||
}
|
||||
if float64(db.count)/float64(db.nBuckets*slotsPerBucket) > loadFactor {
|
||||
if err := db.split(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if db.syncWrites {
|
||||
return db.sync()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) put(hash uint32, key []byte, value []byte) error {
|
||||
var b *bucketHandle
|
||||
var originalB *bucketHandle
|
||||
slotIdx := 0
|
||||
err := db.forEachBucket(db.bucketIndex(hash), func(curb bucketHandle) (bool, error) {
|
||||
b = &curb
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
sl := b.slots[i]
|
||||
slotIdx = i
|
||||
if sl.kvOffset == 0 {
|
||||
// Found an empty slot.
|
||||
return true, nil
|
||||
} else if hash == sl.hash && uint16(len(key)) == sl.keySize {
|
||||
// Key already exists.
|
||||
if slKey, err := db.data.readKey(sl); bytes.Equal(key, slKey) || err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if b.next == 0 {
|
||||
// Couldn't find free space in the current bucketHandle, creating a new overflow bucketHandle.
|
||||
nextBucket, err := db.createOverflowBucket()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
b.next = nextBucket.offset
|
||||
originalB = b
|
||||
b = nextBucket
|
||||
slotIdx = 0
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Inserting a new item.
|
||||
if b.slots[slotIdx].kvOffset == 0 {
|
||||
if db.count == MaxKeys {
|
||||
return errFull
|
||||
}
|
||||
db.count++
|
||||
} else {
|
||||
defer db.data.free(b.slots[slotIdx].kvSize(), b.slots[slotIdx].kvOffset)
|
||||
}
|
||||
|
||||
b.slots[slotIdx] = slot{
|
||||
hash: hash,
|
||||
keySize: uint16(len(key)),
|
||||
valueSize: uint32(len(value)),
|
||||
}
|
||||
if b.slots[slotIdx].kvOffset, err = db.data.writeKeyValue(key, value); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := b.write(); err != nil {
|
||||
return err
|
||||
}
|
||||
if originalB != nil {
|
||||
return originalB.write()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) split() error {
|
||||
updatedBucketIdx := db.splitBucketIdx
|
||||
updatedBucketOff := bucketOffset(updatedBucketIdx)
|
||||
updatedBucket := slotWriter{
|
||||
bucket: &bucketHandle{file: db.index, offset: updatedBucketOff},
|
||||
}
|
||||
|
||||
newBucketOff, err := db.index.extend(bucketSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
newBucket := slotWriter{
|
||||
bucket: &bucketHandle{file: db.index, offset: newBucketOff},
|
||||
}
|
||||
|
||||
db.splitBucketIdx++
|
||||
if db.splitBucketIdx == 1<<db.level {
|
||||
db.level++
|
||||
db.splitBucketIdx = 0
|
||||
}
|
||||
|
||||
var overflowBuckets []int64
|
||||
if err := db.forEachBucket(updatedBucketIdx, func(curb bucketHandle) (bool, error) {
|
||||
for j := 0; j < slotsPerBucket; j++ {
|
||||
sl := curb.slots[j]
|
||||
if sl.kvOffset == 0 {
|
||||
break
|
||||
}
|
||||
if db.bucketIndex(sl.hash) == updatedBucketIdx {
|
||||
if err := updatedBucket.insert(sl, db); err != nil {
|
||||
return true, err
|
||||
}
|
||||
} else {
|
||||
if err := newBucket.insert(sl, db); err != nil {
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if curb.next != 0 {
|
||||
overflowBuckets = append(overflowBuckets, curb.next)
|
||||
}
|
||||
return false, nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, off := range overflowBuckets {
|
||||
db.data.free(bucketSize, off)
|
||||
}
|
||||
|
||||
if err := newBucket.write(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := updatedBucket.write(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
db.nBuckets++
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete deletes the given key from the DB.
|
||||
func (db *DB) Delete(key []byte) error {
|
||||
h := db.hash(key)
|
||||
db.metrics.Dels.Add(1)
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
b := bucketHandle{}
|
||||
slotIdx := -1
|
||||
err := db.forEachBucket(db.bucketIndex(h), func(curb bucketHandle) (bool, error) {
|
||||
b = curb
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
sl := b.slots[i]
|
||||
if sl.kvOffset == 0 {
|
||||
return b.next == 0, nil
|
||||
} else if h == sl.hash && uint16(len(key)) == sl.keySize {
|
||||
slKey, err := db.data.readKey(sl)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
if bytes.Equal(key, slKey) {
|
||||
slotIdx = i
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if slotIdx == -1 || err != nil {
|
||||
return err
|
||||
}
|
||||
sl := b.slots[slotIdx]
|
||||
b.del(slotIdx)
|
||||
if err := b.write(); err != nil {
|
||||
return err
|
||||
}
|
||||
db.data.free(sl.kvSize(), sl.kvOffset)
|
||||
db.count--
|
||||
if db.syncWrites {
|
||||
return db.sync()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Count returns the number of items in the DB.
|
||||
func (db *DB) Count() uint32 {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
return db.count
|
||||
}
|
||||
|
||||
// Metrics returns the DB metrics.
|
||||
func (db *DB) Metrics() Metrics {
|
||||
return db.metrics
|
||||
}
|
||||
|
||||
// FileSize returns the total size of the disk storage used by the DB.
|
||||
func (db *DB) FileSize() (int64, error) {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
var err error
|
||||
is, err := db.index.Stat()
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
ds, err := db.data.Stat()
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
return is.Size() + ds.Size(), nil
|
||||
}
|
||||
4
vendor/github.com/akrylysov/pogreb/doc.go
generated
vendored
Normal file
4
vendor/github.com/akrylysov/pogreb/doc.go
generated
vendored
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
/*
|
||||
Package pogreb implements an embedded key-value store for read-heavy workloads.
|
||||
*/
|
||||
package pogreb
|
||||
14
vendor/github.com/akrylysov/pogreb/errors.go
generated
vendored
Normal file
14
vendor/github.com/akrylysov/pogreb/errors.go
generated
vendored
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
var (
|
||||
errKeyEmpty = errors.New("key is empty")
|
||||
errKeyTooLarge = errors.New("key is too large")
|
||||
errValueTooLarge = errors.New("value is too large")
|
||||
errFull = errors.New("database is full")
|
||||
errCorrupted = errors.New("database is corrupted")
|
||||
errLocked = errors.New("database is locked")
|
||||
)
|
||||
63
vendor/github.com/akrylysov/pogreb/file.go
generated
vendored
Normal file
63
vendor/github.com/akrylysov/pogreb/file.go
generated
vendored
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"encoding"
|
||||
"os"
|
||||
|
||||
"github.com/akrylysov/pogreb/fs"
|
||||
)
|
||||
|
||||
type file struct {
|
||||
fs.MmapFile
|
||||
size int64
|
||||
}
|
||||
|
||||
func openFile(fsyst fs.FileSystem, name string, flag int, perm os.FileMode) (file, error) {
|
||||
fi, err := fsyst.OpenFile(name, flag, perm)
|
||||
f := file{}
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.MmapFile = fi
|
||||
stat, err := fi.Stat()
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
f.size = stat.Size()
|
||||
return f, err
|
||||
}
|
||||
|
||||
func (f *file) extend(size uint32) (int64, error) {
|
||||
off := f.size
|
||||
if err := f.Truncate(off + int64(size)); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
f.size += int64(size)
|
||||
return off, f.Mmap(f.size)
|
||||
}
|
||||
|
||||
func (f *file) append(data []byte) (int64, error) {
|
||||
off := f.size
|
||||
if _, err := f.WriteAt(data, off); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
f.size += int64(len(data))
|
||||
return off, f.Mmap(f.size)
|
||||
}
|
||||
|
||||
func (f *file) writeMarshalableAt(m encoding.BinaryMarshaler, off int64) error {
|
||||
buf, err := m.MarshalBinary()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = f.WriteAt(buf, off)
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *file) readUnmarshalableAt(m encoding.BinaryUnmarshaler, size uint32, off int64) error {
|
||||
buf := make([]byte, size)
|
||||
if _, err := f.ReadAt(buf, off); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.UnmarshalBinary(buf)
|
||||
}
|
||||
149
vendor/github.com/akrylysov/pogreb/freelist.go
generated
vendored
Normal file
149
vendor/github.com/akrylysov/pogreb/freelist.go
generated
vendored
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type block struct {
|
||||
offset int64
|
||||
size uint32
|
||||
}
|
||||
|
||||
type freelist struct {
|
||||
blocks []block
|
||||
}
|
||||
|
||||
func (fl *freelist) search(size uint32) int {
|
||||
return sort.Search(len(fl.blocks), func(i int) bool {
|
||||
return fl.blocks[i].size >= size
|
||||
})
|
||||
}
|
||||
|
||||
func (fl *freelist) free(off int64, size uint32) {
|
||||
if size == 0 {
|
||||
panic("unable to free zero bytes")
|
||||
}
|
||||
i := fl.search(size)
|
||||
if i < len(fl.blocks) && off == fl.blocks[i].offset {
|
||||
panic("freeing already freed offset")
|
||||
}
|
||||
|
||||
fl.blocks = append(fl.blocks, block{})
|
||||
copy(fl.blocks[i+1:], fl.blocks[i:])
|
||||
fl.blocks[i] = block{offset: off, size: size}
|
||||
}
|
||||
|
||||
func (fl *freelist) allocate(size uint32) int64 {
|
||||
if size == 0 {
|
||||
panic("unable to allocate zero bytes")
|
||||
}
|
||||
i := fl.search(size)
|
||||
if i >= len(fl.blocks) {
|
||||
return -1
|
||||
}
|
||||
off := fl.blocks[i].offset
|
||||
if fl.blocks[i].size == size {
|
||||
copy(fl.blocks[i:], fl.blocks[i+1:])
|
||||
fl.blocks[len(fl.blocks)-1] = block{}
|
||||
fl.blocks = fl.blocks[:len(fl.blocks)-1]
|
||||
} else {
|
||||
fl.blocks[i].size -= size
|
||||
fl.blocks[i].offset += int64(size)
|
||||
}
|
||||
return off
|
||||
}
|
||||
|
||||
func (fl *freelist) defrag() {
|
||||
if len(fl.blocks) <= 1 {
|
||||
return
|
||||
}
|
||||
sort.Slice(fl.blocks, func(i, j int) bool {
|
||||
return fl.blocks[i].offset < fl.blocks[j].offset
|
||||
})
|
||||
var merged []block
|
||||
curOff := fl.blocks[0].offset
|
||||
curSize := fl.blocks[0].size
|
||||
for i := 1; i < len(fl.blocks); i++ {
|
||||
if curOff+int64(curSize) == fl.blocks[i].offset {
|
||||
curSize += fl.blocks[i].size
|
||||
} else {
|
||||
merged = append(merged, block{size: curSize, offset: curOff})
|
||||
curOff = fl.blocks[i].offset
|
||||
curSize = fl.blocks[i].size
|
||||
}
|
||||
}
|
||||
merged = append(merged, block{offset: curOff, size: curSize})
|
||||
sort.Slice(merged, func(i, j int) bool {
|
||||
return merged[i].size < merged[j].size
|
||||
})
|
||||
fl.blocks = merged
|
||||
}
|
||||
|
||||
func (fl *freelist) MarshalBinary() ([]byte, error) {
|
||||
size := fl.binarySize()
|
||||
buf := make([]byte, size)
|
||||
data := buf
|
||||
binary.LittleEndian.PutUint32(data[:4], uint32(len(fl.blocks)))
|
||||
data = data[4:]
|
||||
for i := 0; i < len(fl.blocks); i++ {
|
||||
binary.LittleEndian.PutUint64(data[:8], uint64(fl.blocks[i].offset))
|
||||
binary.LittleEndian.PutUint32(data[8:12], fl.blocks[i].size)
|
||||
data = data[12:]
|
||||
}
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func (fl *freelist) binarySize() uint32 {
|
||||
return uint32(4 + (8+4)*len(fl.blocks)) // FIXME: this is ugly
|
||||
}
|
||||
|
||||
func (fl *freelist) read(f file, off int64) error {
|
||||
if off == -1 {
|
||||
return nil
|
||||
}
|
||||
buf := make([]byte, 4)
|
||||
if _, err := f.ReadAt(buf, off); err != nil {
|
||||
return err
|
||||
}
|
||||
n := binary.LittleEndian.Uint32(buf)
|
||||
buf = make([]byte, (4+8)*n)
|
||||
if _, err := f.ReadAt(buf, off+4); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := uint32(0); i < n; i++ {
|
||||
blockOff := int64(binary.LittleEndian.Uint64(buf[:8]))
|
||||
blockSize := binary.LittleEndian.Uint32(buf[8:12])
|
||||
if blockOff != 0 {
|
||||
fl.blocks = append(fl.blocks, block{size: blockSize, offset: blockOff})
|
||||
}
|
||||
buf = buf[12:]
|
||||
}
|
||||
fl.free(off, align512(4+(4+8)*n))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fl *freelist) write(f file) (int64, error) {
|
||||
if len(fl.blocks) == 0 {
|
||||
return -1, nil
|
||||
}
|
||||
marshaledSize := align512(fl.binarySize())
|
||||
i := fl.search(marshaledSize)
|
||||
var off int64
|
||||
if i < len(fl.blocks) {
|
||||
off = fl.blocks[i].offset
|
||||
fl.blocks[i] = block{}
|
||||
} else {
|
||||
var err error
|
||||
off, err = f.extend(marshaledSize)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
}
|
||||
buf, err := fl.MarshalBinary()
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
_, err = f.WriteAt(buf, off)
|
||||
return off, err
|
||||
}
|
||||
40
vendor/github.com/akrylysov/pogreb/fs/file.go
generated
vendored
Normal file
40
vendor/github.com/akrylysov/pogreb/fs/file.go
generated
vendored
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// File is the interface compatible with os.File.
|
||||
type File interface {
|
||||
io.Closer
|
||||
io.Reader
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
io.Writer
|
||||
io.WriterAt
|
||||
|
||||
Stat() (os.FileInfo, error)
|
||||
Sync() error
|
||||
Truncate(size int64) error
|
||||
}
|
||||
|
||||
// MmapFile represents a memory-mapped file.
|
||||
type MmapFile interface {
|
||||
File
|
||||
Mmap(size int64) error
|
||||
Slice(start int64, end int64) ([]byte, error)
|
||||
}
|
||||
|
||||
// LockFile represents a lock file.
|
||||
type LockFile interface {
|
||||
Unlock() error
|
||||
}
|
||||
|
||||
// FileSystem represents a virtual file system.
|
||||
type FileSystem interface {
|
||||
OpenFile(name string, flag int, perm os.FileMode) (MmapFile, error)
|
||||
CreateLockFile(name string, perm os.FileMode) (LockFile, bool, error)
|
||||
Stat(name string) (os.FileInfo, error)
|
||||
Remove(name string) error
|
||||
}
|
||||
201
vendor/github.com/akrylysov/pogreb/fs/mem.go
generated
vendored
Normal file
201
vendor/github.com/akrylysov/pogreb/fs/mem.go
generated
vendored
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type memfs struct {
|
||||
files map[string]*memfile
|
||||
}
|
||||
|
||||
// Mem is a file system backed by memory.
|
||||
var Mem = &memfs{files: map[string]*memfile{}}
|
||||
|
||||
func (fs *memfs) OpenFile(name string, flag int, perm os.FileMode) (MmapFile, error) {
|
||||
// TODO: respect flag
|
||||
f := fs.files[name]
|
||||
if f == nil {
|
||||
f = &memfile{}
|
||||
fs.files[name] = f
|
||||
} else if !f.closed {
|
||||
return nil, os.ErrExist
|
||||
} else {
|
||||
f.closed = false
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (fs *memfs) CreateLockFile(name string, perm os.FileMode) (LockFile, bool, error) {
|
||||
f, err := fs.OpenFile(name, 0, perm)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return &memlockfile{f, name}, false, nil
|
||||
}
|
||||
|
||||
func (fs *memfs) Stat(name string) (os.FileInfo, error) {
|
||||
if f, ok := fs.files[name]; ok {
|
||||
return f, nil
|
||||
}
|
||||
return nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
func (fs *memfs) Remove(name string) error {
|
||||
if _, ok := fs.files[name]; ok {
|
||||
delete(fs.files, name)
|
||||
return nil
|
||||
}
|
||||
return os.ErrNotExist
|
||||
}
|
||||
|
||||
type memlockfile struct {
|
||||
File
|
||||
name string
|
||||
}
|
||||
|
||||
func (f *memlockfile) Unlock() error {
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return Mem.Remove(f.name)
|
||||
}
|
||||
|
||||
type memfile struct {
|
||||
buf []byte
|
||||
size int64
|
||||
offset int64
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (m *memfile) Close() error {
|
||||
if m.closed {
|
||||
return os.ErrClosed
|
||||
}
|
||||
m.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memfile) ReadAt(p []byte, off int64) (int, error) {
|
||||
if m.closed {
|
||||
return 0, os.ErrClosed
|
||||
}
|
||||
n := len(p)
|
||||
if int64(n) > m.size-off {
|
||||
return 0, io.EOF
|
||||
}
|
||||
copy(p, m.buf[off:off+int64(n)])
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (m *memfile) Read(p []byte) (int, error) {
|
||||
n, err := m.ReadAt(p, m.offset)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
m.offset += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (m *memfile) WriteAt(p []byte, off int64) (int, error) {
|
||||
if m.closed {
|
||||
return 0, os.ErrClosed
|
||||
}
|
||||
n := len(p)
|
||||
if off == m.size {
|
||||
m.buf = append(m.buf, p...)
|
||||
m.size += int64(n)
|
||||
} else if off+int64(n) > m.size {
|
||||
panic("trying to write past EOF - undefined behavior")
|
||||
} else {
|
||||
copy(m.buf[off:off+int64(n)], p)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (m *memfile) Write(p []byte) (int, error) {
|
||||
n, err := m.WriteAt(p, m.offset)
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
m.offset += int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (m *memfile) Seek(offset int64, whence int) (int64, error) {
|
||||
if m.closed {
|
||||
return 0, os.ErrClosed
|
||||
}
|
||||
if whence == io.SeekEnd {
|
||||
m.offset = m.size + offset
|
||||
} else if whence == io.SeekStart {
|
||||
m.offset = offset
|
||||
} else if whence == io.SeekCurrent {
|
||||
m.offset += offset
|
||||
}
|
||||
return m.offset, nil
|
||||
}
|
||||
|
||||
func (m *memfile) Stat() (os.FileInfo, error) {
|
||||
if m.closed {
|
||||
return m, os.ErrClosed
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
func (m *memfile) Sync() error {
|
||||
if m.closed {
|
||||
return os.ErrClosed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memfile) Truncate(size int64) error {
|
||||
if m.closed {
|
||||
return os.ErrClosed
|
||||
}
|
||||
if size > m.size {
|
||||
diff := int(size - m.size)
|
||||
m.buf = append(m.buf, make([]byte, diff)...)
|
||||
} else {
|
||||
m.buf = m.buf[:m.size]
|
||||
}
|
||||
m.size = size
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memfile) Name() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *memfile) Size() int64 {
|
||||
return m.size
|
||||
}
|
||||
|
||||
func (m *memfile) Mode() os.FileMode {
|
||||
return os.FileMode(0)
|
||||
}
|
||||
|
||||
func (m *memfile) ModTime() time.Time {
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (m *memfile) IsDir() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *memfile) Sys() interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *memfile) Slice(start int64, end int64) ([]byte, error) {
|
||||
if m.closed {
|
||||
return nil, os.ErrClosed
|
||||
}
|
||||
return m.buf[start:end], nil
|
||||
}
|
||||
|
||||
func (m *memfile) Mmap(size int64) error {
|
||||
return nil
|
||||
}
|
||||
110
vendor/github.com/akrylysov/pogreb/fs/os.go
generated
vendored
Normal file
110
vendor/github.com/akrylysov/pogreb/fs/os.go
generated
vendored
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
)
|
||||
|
||||
const (
|
||||
initialMmapSize = 1024 << 20
|
||||
)
|
||||
|
||||
type osfile struct {
|
||||
*os.File
|
||||
data []byte
|
||||
mmapSize int64
|
||||
}
|
||||
|
||||
type osfs struct{}
|
||||
|
||||
// OS is a file system backed by the os package.
|
||||
var OS = &osfs{}
|
||||
|
||||
func (fs *osfs) OpenFile(name string, flag int, perm os.FileMode) (MmapFile, error) {
|
||||
f, err := os.OpenFile(name, flag, perm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mf := &osfile{f, nil, 0}
|
||||
if stat.Size() > 0 {
|
||||
if err := mf.Mmap(stat.Size()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return mf, err
|
||||
}
|
||||
|
||||
func (fs *osfs) CreateLockFile(name string, perm os.FileMode) (LockFile, bool, error) {
|
||||
return createLockFile(name, perm)
|
||||
}
|
||||
|
||||
func (fs *osfs) Stat(name string) (os.FileInfo, error) {
|
||||
return os.Stat(name)
|
||||
}
|
||||
|
||||
func (fs *osfs) Remove(name string) error {
|
||||
return os.Remove(name)
|
||||
}
|
||||
|
||||
type oslockfile struct {
|
||||
File
|
||||
path string
|
||||
}
|
||||
|
||||
func (f *oslockfile) Unlock() error {
|
||||
if err := os.Remove(f.path); err != nil {
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
func (f *osfile) Slice(start int64, end int64) ([]byte, error) {
|
||||
if f.data == nil {
|
||||
return nil, os.ErrClosed
|
||||
}
|
||||
return f.data[start:end], nil
|
||||
}
|
||||
|
||||
func (f *osfile) Close() error {
|
||||
if f.data != nil {
|
||||
if err := munmap(f.data); err != nil {
|
||||
return nil
|
||||
}
|
||||
f.data = nil
|
||||
}
|
||||
return f.File.Close()
|
||||
}
|
||||
|
||||
func (f *osfile) Mmap(fileSize int64) error {
|
||||
mmapSize := f.mmapSize
|
||||
|
||||
if mmapSize >= fileSize {
|
||||
return nil
|
||||
}
|
||||
|
||||
if mmapSize == 0 {
|
||||
mmapSize = initialMmapSize
|
||||
if mmapSize < fileSize {
|
||||
mmapSize = fileSize
|
||||
}
|
||||
} else {
|
||||
if err := munmap(f.data); err != nil {
|
||||
return err
|
||||
}
|
||||
mmapSize *= 2
|
||||
}
|
||||
|
||||
data, mappedSize, err := mmap(f.File, fileSize, mmapSize)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
madviceRandom(data)
|
||||
|
||||
f.data = data
|
||||
f.mmapSize = mappedSize
|
||||
return nil
|
||||
}
|
||||
44
vendor/github.com/akrylysov/pogreb/fs/os_unix.go
generated
vendored
Normal file
44
vendor/github.com/akrylysov/pogreb/fs/os_unix.go
generated
vendored
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
// +build !windows
|
||||
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func mmap(f *os.File, fileSize int64, mmapSize int64) ([]byte, int64, error) {
|
||||
p, err := syscall.Mmap(int(f.Fd()), 0, int(mmapSize), syscall.PROT_READ, syscall.MAP_SHARED)
|
||||
return p, mmapSize, err
|
||||
}
|
||||
|
||||
func munmap(data []byte) error {
|
||||
return syscall.Munmap(data)
|
||||
}
|
||||
|
||||
func madviceRandom(data []byte) error {
|
||||
_, _, errno := syscall.Syscall(syscall.SYS_MADVISE, uintptr(unsafe.Pointer(&data[0])), uintptr(len(data)), uintptr(syscall.MADV_RANDOM))
|
||||
if errno != 0 {
|
||||
return errno
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createLockFile(name string, perm os.FileMode) (LockFile, bool, error) {
|
||||
acquiredExisting := false
|
||||
if _, err := os.Stat(name); err == nil {
|
||||
acquiredExisting = true
|
||||
}
|
||||
f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, perm)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||||
if err == syscall.EWOULDBLOCK {
|
||||
err = os.ErrExist
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
return &oslockfile{f, name}, acquiredExisting, nil
|
||||
}
|
||||
84
vendor/github.com/akrylysov/pogreb/fs/os_windows.go
generated
vendored
Normal file
84
vendor/github.com/akrylysov/pogreb/fs/os_windows.go
generated
vendored
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
// +build windows
|
||||
|
||||
package fs
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
var (
|
||||
modkernel32 = syscall.NewLazyDLL("kernel32.dll")
|
||||
procLockFileEx = modkernel32.NewProc("LockFileEx")
|
||||
)
|
||||
|
||||
const (
|
||||
errorLockViolation = 0x21
|
||||
)
|
||||
|
||||
func mmap(f *os.File, fileSize int64, mmapSize int64) ([]byte, int64, error) {
|
||||
size := fileSize
|
||||
low, high := uint32(size), uint32(size>>32)
|
||||
fmap, err := syscall.CreateFileMapping(syscall.Handle(f.Fd()), nil, syscall.PAGE_READONLY, high, low, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer syscall.CloseHandle(fmap)
|
||||
ptr, err := syscall.MapViewOfFile(fmap, syscall.FILE_MAP_READ, 0, 0, uintptr(size))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
data := (*[1 << 30]byte)(unsafe.Pointer(ptr))[:size]
|
||||
return data, size, nil
|
||||
}
|
||||
|
||||
func munmap(data []byte) error {
|
||||
return syscall.UnmapViewOfFile(uintptr(unsafe.Pointer(&data[0])))
|
||||
}
|
||||
|
||||
func madviceRandom(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockfile(f *os.File) error {
|
||||
var ol syscall.Overlapped
|
||||
|
||||
r1, _, err := syscall.Syscall6(
|
||||
procLockFileEx.Addr(),
|
||||
6,
|
||||
uintptr(f.Fd()), // handle
|
||||
uintptr(0x0003),
|
||||
uintptr(0), // reserved
|
||||
uintptr(1), // locklow
|
||||
uintptr(0), // lockhigh
|
||||
uintptr(unsafe.Pointer(&ol)),
|
||||
)
|
||||
if r1 == 0 && (err == syscall.ERROR_FILE_EXISTS || err == errorLockViolation) {
|
||||
return os.ErrExist
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createLockFile(name string, perm os.FileMode) (LockFile, bool, error) {
|
||||
acquiredExisting := false
|
||||
if _, err := os.Stat(name); err == nil {
|
||||
acquiredExisting = true
|
||||
}
|
||||
fd, err := syscall.CreateFile(&(syscall.StringToUTF16(name)[0]),
|
||||
syscall.GENERIC_READ|syscall.GENERIC_WRITE,
|
||||
syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE,
|
||||
nil,
|
||||
syscall.CREATE_ALWAYS,
|
||||
syscall.FILE_ATTRIBUTE_NORMAL,
|
||||
0)
|
||||
if err != nil {
|
||||
return nil, false, os.ErrExist
|
||||
}
|
||||
f := os.NewFile(uintptr(fd), name)
|
||||
if err := lockfile(f); err != nil {
|
||||
f.Close()
|
||||
return nil, false, err
|
||||
}
|
||||
return &oslockfile{f, name}, acquiredExisting, nil
|
||||
}
|
||||
3
vendor/github.com/akrylysov/pogreb/go.mod
generated
vendored
Normal file
3
vendor/github.com/akrylysov/pogreb/go.mod
generated
vendored
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
module github.com/akrylysov/pogreb
|
||||
|
||||
go 1.12
|
||||
61
vendor/github.com/akrylysov/pogreb/hash/murmurhash32.go
generated
vendored
Normal file
61
vendor/github.com/akrylysov/pogreb/hash/murmurhash32.go
generated
vendored
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
package hash
|
||||
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
const (
|
||||
c1 uint32 = 0xcc9e2d51
|
||||
c2 uint32 = 0x1b873593
|
||||
)
|
||||
|
||||
// Sum32WithSeed is a port of MurmurHash3_x86_32 function.
|
||||
func Sum32WithSeed(data []byte, seed uint32) uint32 {
|
||||
h1 := seed
|
||||
|
||||
nblocks := len(data) / 4
|
||||
var p uintptr
|
||||
if len(data) > 0 {
|
||||
p = uintptr(unsafe.Pointer(&data[0]))
|
||||
}
|
||||
p1 := p + uintptr(4*nblocks)
|
||||
for ; p < p1; p += 4 {
|
||||
k1 := *(*uint32)(unsafe.Pointer(p))
|
||||
|
||||
k1 *= c1
|
||||
k1 = (k1 << 15) | (k1 >> 17) // rotl32(k1, 15)
|
||||
k1 *= c2
|
||||
|
||||
h1 ^= k1
|
||||
h1 = (h1 << 13) | (h1 >> 19) // rotl32(h1, 13)
|
||||
h1 = h1*5 + 0xe6546b64
|
||||
}
|
||||
|
||||
tail := data[nblocks*4:]
|
||||
|
||||
var k1 uint32
|
||||
switch len(tail) & 3 {
|
||||
case 3:
|
||||
k1 ^= uint32(tail[2]) << 16
|
||||
fallthrough
|
||||
case 2:
|
||||
k1 ^= uint32(tail[1]) << 8
|
||||
fallthrough
|
||||
case 1:
|
||||
k1 ^= uint32(tail[0])
|
||||
k1 *= c1
|
||||
k1 = (k1 << 15) | (k1 >> 17) // rotl32(k1, 15)
|
||||
k1 *= c2
|
||||
h1 ^= k1
|
||||
}
|
||||
|
||||
h1 ^= uint32(len(data))
|
||||
|
||||
h1 ^= h1 >> 16
|
||||
h1 *= 0x85ebca6b
|
||||
h1 ^= h1 >> 13
|
||||
h1 *= 0xc2b2ae35
|
||||
h1 ^= h1 >> 16
|
||||
|
||||
return h1
|
||||
}
|
||||
15
vendor/github.com/akrylysov/pogreb/hash/seed.go
generated
vendored
Normal file
15
vendor/github.com/akrylysov/pogreb/hash/seed.go
generated
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
package hash
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// RandSeed generates a random hash seed.
|
||||
func RandSeed() (uint32, error) {
|
||||
b := make([]byte, 4)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return binary.LittleEndian.Uint32(b), nil
|
||||
}
|
||||
46
vendor/github.com/akrylysov/pogreb/header.go
generated
vendored
Normal file
46
vendor/github.com/akrylysov/pogreb/header.go
generated
vendored
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
var (
|
||||
signature = [8]byte{'p', 'o', 'g', 'r', 'e', 'b', '\x0e', '\xfd'}
|
||||
headerSize uint32
|
||||
)
|
||||
|
||||
type header struct {
|
||||
signature [8]byte
|
||||
version uint32
|
||||
dbInfo
|
||||
_ [256]byte
|
||||
}
|
||||
|
||||
func init() {
|
||||
headerSize = align512(uint32(binary.Size(header{})))
|
||||
}
|
||||
|
||||
func (h header) MarshalBinary() ([]byte, error) {
|
||||
buf := make([]byte, headerSize)
|
||||
copy(buf[:8], h.signature[:])
|
||||
binary.LittleEndian.PutUint32(buf[8:12], h.version)
|
||||
buf[12] = h.level
|
||||
binary.LittleEndian.PutUint32(buf[13:17], h.count)
|
||||
binary.LittleEndian.PutUint32(buf[17:21], h.nBuckets)
|
||||
binary.LittleEndian.PutUint32(buf[21:25], h.splitBucketIdx)
|
||||
binary.LittleEndian.PutUint64(buf[25:33], uint64(h.freelistOff))
|
||||
binary.LittleEndian.PutUint32(buf[33:37], h.hashSeed)
|
||||
return buf, nil
|
||||
}
|
||||
|
||||
func (h *header) UnmarshalBinary(data []byte) error {
|
||||
copy(h.signature[:], data[:8])
|
||||
h.version = binary.LittleEndian.Uint32(data[8:12])
|
||||
h.level = data[12]
|
||||
h.count = binary.LittleEndian.Uint32(data[13:17])
|
||||
h.nBuckets = binary.LittleEndian.Uint32(data[17:21])
|
||||
h.splitBucketIdx = binary.LittleEndian.Uint32(data[21:25])
|
||||
h.freelistOff = int64(binary.LittleEndian.Uint64(data[25:33]))
|
||||
h.hashSeed = binary.LittleEndian.Uint32(data[33:37])
|
||||
return nil
|
||||
}
|
||||
65
vendor/github.com/akrylysov/pogreb/iterator.go
generated
vendored
Normal file
65
vendor/github.com/akrylysov/pogreb/iterator.go
generated
vendored
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// ErrIterationDone is returned by ItemIterator.Next calls when there are no more items to return.
|
||||
var ErrIterationDone = errors.New("no more items in iterator")
|
||||
|
||||
type item struct {
|
||||
key []byte
|
||||
value []byte
|
||||
}
|
||||
|
||||
// ItemIterator is an iterator over DB key/value pairs. It iterates the items in an unspecified order.
|
||||
type ItemIterator struct {
|
||||
db *DB
|
||||
nextBucketIdx uint32
|
||||
queue []item
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
// Next returns the next key/value pair if available, otherwise it returns ErrIterationDone error.
|
||||
func (it *ItemIterator) Next() ([]byte, []byte, error) {
|
||||
it.mu.Lock()
|
||||
defer it.mu.Unlock()
|
||||
|
||||
it.db.mu.RLock()
|
||||
defer it.db.mu.RUnlock()
|
||||
|
||||
if len(it.queue) == 0 {
|
||||
for it.nextBucketIdx < it.db.nBuckets {
|
||||
err := it.db.forEachBucket(it.nextBucketIdx, func(b bucketHandle) (bool, error) {
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
sl := b.slots[i]
|
||||
if sl.kvOffset == 0 {
|
||||
return true, nil
|
||||
}
|
||||
key, value, err := it.db.data.readKeyValue(sl)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
it.queue = append(it.queue, item{key: key, value: value})
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
it.nextBucketIdx++
|
||||
if len(it.queue) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(it.queue) > 0 {
|
||||
item := it.queue[0]
|
||||
it.queue = it.queue[1:]
|
||||
return item.key, item.value, nil
|
||||
}
|
||||
|
||||
return nil, nil, ErrIterationDone
|
||||
}
|
||||
15
vendor/github.com/akrylysov/pogreb/logger.go
generated
vendored
Normal file
15
vendor/github.com/akrylysov/pogreb/logger.go
generated
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
)
|
||||
|
||||
var logger = log.New(os.Stderr, "pogreb: ", 0)
|
||||
|
||||
// SetLogger sets the global logger.
|
||||
func SetLogger(l *log.Logger) {
|
||||
if l != nil {
|
||||
logger = l
|
||||
}
|
||||
}
|
||||
22
vendor/github.com/akrylysov/pogreb/metrics.go
generated
vendored
Normal file
22
vendor/github.com/akrylysov/pogreb/metrics.go
generated
vendored
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
package pogreb
|
||||
|
||||
import "expvar"
|
||||
|
||||
// Metrics holds the DB metrics.
|
||||
type Metrics struct {
|
||||
Puts *expvar.Int
|
||||
Dels *expvar.Int
|
||||
Gets *expvar.Int
|
||||
HashCollisions *expvar.Int
|
||||
BucketProbes *expvar.Int
|
||||
}
|
||||
|
||||
func newMetrics() Metrics {
|
||||
return Metrics{
|
||||
Puts: &expvar.Int{},
|
||||
Dels: &expvar.Int{},
|
||||
Gets: &expvar.Int{},
|
||||
HashCollisions: &expvar.Int{},
|
||||
BucketProbes: &expvar.Int{},
|
||||
}
|
||||
}
|
||||
29
vendor/github.com/akrylysov/pogreb/options.go
generated
vendored
Normal file
29
vendor/github.com/akrylysov/pogreb/options.go
generated
vendored
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/akrylysov/pogreb/fs"
|
||||
)
|
||||
|
||||
// Options holds the optional DB parameters.
|
||||
type Options struct {
|
||||
// BackgroundSyncInterval sets the amount of time between background fsync() calls.
|
||||
//
|
||||
// Setting the value to 0 disables the automatic background synchronization.
|
||||
// Setting the value to -1 makes the DB call fsync() after every write operation.
|
||||
BackgroundSyncInterval time.Duration
|
||||
|
||||
FileSystem fs.FileSystem
|
||||
}
|
||||
|
||||
func (src *Options) copyWithDefaults() *Options {
|
||||
opts := Options{}
|
||||
if src != nil {
|
||||
opts = *src
|
||||
}
|
||||
if opts.FileSystem == nil {
|
||||
opts.FileSystem = fs.OS
|
||||
}
|
||||
return &opts
|
||||
}
|
||||
154
vendor/github.com/akrylysov/pogreb/recovery.go
generated
vendored
Normal file
154
vendor/github.com/akrylysov/pogreb/recovery.go
generated
vendored
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
package pogreb
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
func align51264(n int64) int64 {
|
||||
return (n + 511) &^ 511
|
||||
}
|
||||
|
||||
func truncateFiles(db *DB) error {
|
||||
db.index.size = align51264(db.index.size)
|
||||
if err := db.index.Truncate(db.index.size); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.index.Mmap(db.index.size); err != nil {
|
||||
return err
|
||||
}
|
||||
db.data.size = align51264(db.data.size)
|
||||
if err := db.data.Truncate(db.data.size); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.data.Mmap(db.data.size); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getUsedBlocks(db *DB) (uint32, []block, error) {
|
||||
var itemCount uint32
|
||||
var usedBlocks []block
|
||||
for bucketIdx := uint32(0); bucketIdx < db.nBuckets; bucketIdx++ {
|
||||
err := db.forEachBucket(bucketIdx, func(b bucketHandle) (bool, error) {
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
sl := b.slots[i]
|
||||
if sl.kvOffset == 0 {
|
||||
return true, nil
|
||||
}
|
||||
itemCount++
|
||||
usedBlocks = append(usedBlocks, block{size: align512(sl.kvSize()), offset: sl.kvOffset})
|
||||
}
|
||||
if b.next != 0 {
|
||||
usedBlocks = append(usedBlocks, block{size: bucketSize, offset: b.next})
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
}
|
||||
return itemCount, usedBlocks, nil
|
||||
}
|
||||
|
||||
func recoverSplitCrash(db *DB) error {
|
||||
if db.nBuckets == 1 {
|
||||
return nil
|
||||
}
|
||||
prevnBuckets := db.nBuckets - 1
|
||||
prevLevel := uint8(math.Floor(math.Log2(float64(prevnBuckets))))
|
||||
prevSplitBucketIdx := prevnBuckets - (uint32(1) << prevLevel)
|
||||
splitCrash := false
|
||||
err := db.forEachBucket(prevSplitBucketIdx, func(b bucketHandle) (bool, error) {
|
||||
for i := 0; i < slotsPerBucket; i++ {
|
||||
sl := b.slots[i]
|
||||
if sl.kvOffset == 0 {
|
||||
return true, nil
|
||||
}
|
||||
if db.bucketIndex(sl.hash) != prevSplitBucketIdx {
|
||||
splitCrash = true
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !splitCrash {
|
||||
return nil
|
||||
}
|
||||
logger.Print("Detected split crash. Truncating index file...")
|
||||
if err := db.index.Truncate(db.index.size - int64(bucketSize)); err != nil {
|
||||
return err
|
||||
}
|
||||
db.index.size -= int64(bucketSize)
|
||||
if err := db.index.Mmap(db.index.size); err != nil {
|
||||
return err
|
||||
}
|
||||
db.nBuckets = prevnBuckets
|
||||
db.level = prevLevel
|
||||
db.splitBucketIdx = prevSplitBucketIdx
|
||||
return nil
|
||||
}
|
||||
|
||||
func recoverFreeList(db *DB, usedBlocks []block) error {
|
||||
if len(usedBlocks) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(usedBlocks, func(i, j int) bool {
|
||||
return usedBlocks[i].offset < usedBlocks[j].offset
|
||||
})
|
||||
fl := freelist{}
|
||||
expectedOff := int64(headerSize)
|
||||
for _, bl := range usedBlocks {
|
||||
if bl.offset > expectedOff {
|
||||
fl.free(expectedOff, uint32(bl.offset-expectedOff))
|
||||
}
|
||||
expectedOff = bl.offset + int64(bl.size)
|
||||
}
|
||||
lastBlock := usedBlocks[len(usedBlocks)-1]
|
||||
lastOffset := int64(lastBlock.size) + lastBlock.offset
|
||||
if db.data.size > lastOffset {
|
||||
fl.free(lastOffset, uint32(db.data.size-lastOffset))
|
||||
logger.Println(lastBlock, db.data.size)
|
||||
}
|
||||
logger.Printf("Recovered freelist. Old len=%d; new len=%d\n", len(db.data.fl.blocks), len(fl.blocks))
|
||||
db.data.fl = fl
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) recover() error {
|
||||
logger.Println("Performing recovery...")
|
||||
logger.Printf("Index file size=%d; data file size=%d\n", db.index.size, db.data.size)
|
||||
logger.Printf("Header dbInfo %+v\n", db.dbInfo)
|
||||
|
||||
// Truncate index and data files.
|
||||
if err := truncateFiles(db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Recover header.
|
||||
db.nBuckets = uint32((db.index.size - int64(headerSize)) / int64(bucketSize))
|
||||
db.level = uint8(math.Floor(math.Log2(float64(db.nBuckets))))
|
||||
db.splitBucketIdx = db.nBuckets - (uint32(1) << db.level)
|
||||
itemCount, usedBlocks, err := getUsedBlocks(db)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
db.count = itemCount
|
||||
|
||||
// Check if crash occurred during split.
|
||||
if err := recoverSplitCrash(db); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Printf("Recovered dbInfo %+v\n", db.dbInfo)
|
||||
|
||||
// Recover free list.
|
||||
if err := recoverFreeList(db, usedBlocks); err != nil {
|
||||
return err
|
||||
}
|
||||
logger.Println("Recovery complete.")
|
||||
return nil
|
||||
}
|
||||
18
vendor/vendor.json
vendored
18
vendor/vendor.json
vendored
|
|
@ -20,6 +20,24 @@
|
|||
"revision": "5d049714c4a64225c3c79a7cf7d02f7fb5b96338",
|
||||
"revisionTime": "2018-01-16T20:38:02Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "EBi6awUmT1usWPTFQseEzp70578=",
|
||||
"path": "github.com/akrylysov/pogreb",
|
||||
"revision": "828d0ef698122ac786f0c769eeea8c2b41ac1493",
|
||||
"revisionTime": "2019-06-30T21:56:05Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "7/INULWANAgDLXBI4eLIOgjjpzg=",
|
||||
"path": "github.com/akrylysov/pogreb/fs",
|
||||
"revision": "828d0ef698122ac786f0c769eeea8c2b41ac1493",
|
||||
"revisionTime": "2019-06-30T21:56:05Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "ykjZaRe9OBpXJFXqjlzInpzPM24=",
|
||||
"path": "github.com/akrylysov/pogreb/hash",
|
||||
"revision": "828d0ef698122ac786f0c769eeea8c2b41ac1493",
|
||||
"revisionTime": "2019-06-30T21:56:05Z"
|
||||
},
|
||||
{
|
||||
"checksumSHA1": "8skJYOdQytXjimcDPLRW4tonX3A=",
|
||||
"path": "github.com/allegro/bigcache",
|
||||
|
|
|
|||
Loading…
Reference in a new issue