diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index f29c18d7c0..6963fb3c1d 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -76,7 +76,6 @@ const ( SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" - SWARM_ENV_STORE_RADIUS = "SWARM_STORE_RADIUS" GETH_ENV_DATADIR = "GETH_DATADIR" ) @@ -237,19 +236,15 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con } if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" { - currentConfig.StoreParams.ChunkDbPath = storePath + currentConfig.LocalStoreParams.ChunkDbPath = storePath } if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 { - currentConfig.StoreParams.DbCapacity = storeCapacity + currentConfig.LocalStoreParams.DbCapacity = storeCapacity } if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 { - currentConfig.StoreParams.CacheCapacity = storeCacheCapacity - } - - if storeRadius := ctx.GlobalInt(SwarmStoreRadius.Name); storeRadius != 0 { - currentConfig.StoreParams.Radius = storeRadius + currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity } return currentConfig diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index 266e0a7146..42d9a62062 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -156,7 +156,7 @@ func TestConfigFileOverrides(t *testing.T) { defaultConf.PssEnabled = true defaultConf.NetworkId = 54 defaultConf.Port = httpPort - defaultConf.StoreParams.DbCapacity = 9000000 + defaultConf.DbCapacity = 9000000 defaultConf.HiveParams.KeepAliveInterval = 6000000000 defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second //defaultConf.SyncParams.KeyBufferSize = 512 @@ -232,7 +232,7 @@ func TestConfigFileOverrides(t *testing.T) { t.Fatal("Expected Pss to be enabled, but is false") } - if info.StoreParams.DbCapacity != 9000000 { + if info.DbCapacity != 9000000 { t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId) } @@ -368,7 +368,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) { defaultConf.PssEnabled = false defaultConf.NetworkId = 54 defaultConf.Port = "8588" - defaultConf.StoreParams.DbCapacity = 9000000 + defaultConf.DbCapacity = 9000000 defaultConf.HiveParams.KeepAliveInterval = 6000000000 defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second //defaultConf.SyncParams.KeyBufferSize = 512 @@ -378,10 +378,12 @@ func TestConfigCmdLineOverridesFile(t *testing.T) { t.Fatalf("Error creating TOML file in TestFileOverride: %v", err) } //write file - f, err := ioutil.TempFile("", "testconfig.toml") + fname := "testconfig.toml" + f, err := ioutil.TempFile("", fname) if err != nil { t.Fatalf("Error writing TOML file in TestFileOverride: %v", err) } + defer os.Remove(fname) //write file _, err = f.WriteString(string(out)) if err != nil { @@ -446,8 +448,8 @@ func TestConfigCmdLineOverridesFile(t *testing.T) { t.Fatal("Expected Sync to be enabled, but is false") } - if info.StoreParams.DbCapacity != 9000000 { - t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId) + if info.LocalStoreParams.DbCapacity != 9000000 { + t.Fatalf("Expected Capacity to be %d, got %d", 9000000, info.LocalStoreParams.DbCapacity) } if info.HiveParams.KeepAliveInterval != 6000000000 { diff --git a/cmd/swarm/db.go b/cmd/swarm/db.go index eee5d25cb9..a080be7f57 100644 --- a/cmd/swarm/db.go +++ b/cmd/swarm/db.go @@ -112,6 +112,8 @@ func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) { if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil { return nil, fmt.Errorf("invalid chunkdb path: %s", err) } - hash := storage.MakeHashFunc("SHA3") - return storage.NewLDBStore(path, hash, 10000000, func(k storage.Key) (ret uint8) { return uint8(storage.Proximity(basekey[:], k[:])) }) + + storeparams := storage.NewDefaultStoreParams() + ldbparams := storage.NewLDBStoreParams(storeparams, path) + return storage.NewLDBStore(ldbparams) } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 8ca1388447..5570a9fde7 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -169,11 +169,6 @@ var ( Usage: "Number of recent chunks cached in memory (default 5000)", EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY, } - SwarmStoreRadius = cli.IntFlag{ - Name: "store.radius", - Usage: "Minimum proximity order (number of identical prefix bits of address key) for chunks to warrant storage (default 0)", - EnvVar: SWARM_ENV_STORE_RADIUS, - } // the following flags are deprecated and should be removed in the future DeprecatedEthAPIFlag = cli.StringFlag{ @@ -385,7 +380,6 @@ Remove corrupt entries from a local chunk database. SwarmStorePath, SwarmStoreCapacity, SwarmStoreCacheCapacity, - SwarmStoreRadius, //deprecated flags DeprecatedEthAPIFlag, DeprecatedEnsAddrFlag, diff --git a/swarm/api/api.go b/swarm/api/api.go index 8b48419e44..1e08b71d35 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -617,7 +617,7 @@ func (self *Api) ResourceUpdate(ctx context.Context, name string, data []byte) ( } func (self *Api) ResourceHashSize() int { - return self.resource.HashSize() + return self.resource.HashSize } func (self *Api) ResourceIsValidated() bool { diff --git a/swarm/api/config.go b/swarm/api/config.go index 3f93f8f930..8f608d6e44 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -42,8 +42,8 @@ const ( // allow several bzz nodes running in parallel type Config struct { // serialised/persisted fields - *storage.StoreParams *storage.DPAParams + *storage.LocalStoreParams *network.HiveParams Swap *swap.SwapParams //*network.SyncParams @@ -72,9 +72,9 @@ type Config struct { func NewConfig() (self *Config) { self = &Config{ - StoreParams: storage.NewDefaultStoreParams(), - DPAParams: storage.NewDPAParams(), - HiveParams: network.NewHiveParams(), + LocalStoreParams: storage.NewDefaultLocalStoreParams(), + DPAParams: storage.NewDPAParams(), + HiveParams: network.NewHiveParams(), //SyncParams: network.NewDefaultSyncParams(), Swap: swap.NewDefaultSwapParams(), ListenAddr: DefaultHTTPListenAddr, @@ -117,8 +117,9 @@ func (self *Config) Init(prvKey *ecdsa.PrivateKey) { if self.SwapEnabled { self.Swap.Init(self.Contract, prvKey) } + self.privateKey = prvKey - self.StoreParams.Init(self.Path) + self.LocalStoreParams.Init(self.Path) } func (self *Config) ShiftPrivateKey() (privKey *ecdsa.PrivateKey) { diff --git a/swarm/api/config_test.go b/swarm/api/config_test.go index 5a5f176c0b..bd7e1d8705 100644 --- a/swarm/api/config_test.go +++ b/swarm/api/config_test.go @@ -36,6 +36,7 @@ func TestConfig(t *testing.T) { one := NewConfig() two := NewConfig() + one.LocalStoreParams = two.LocalStoreParams if equal := reflect.DeepEqual(one, two); !equal { t.Fatal("Two default configs are not equal") } @@ -52,7 +53,7 @@ func TestConfig(t *testing.T) { if one.Swap.PayProfile.Beneficiary == (common.Address{}) && one.SwapEnabled { t.Fatal("Failed to correctly initialize SwapParams") } - if one.StoreParams.ChunkDbPath == one.Path { + if one.ChunkDbPath == one.Path { t.Fatal("Failed to correctly initialize StoreParams") } } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 9dc314a199..4d93c7362d 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -481,13 +481,13 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr code := 0 defaultErr := fmt.Errorf("%s: %v", supErr, err) rsrcErr, ok := err.(*storage.ResourceError) - if !ok { + if !ok && rsrcErr != nil { code = rsrcErr.Code() } switch code { case storage.ErrInvalidValue: return http.StatusBadRequest, defaultErr - case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn: + case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn, storage.ErrInit: return http.StatusNotFound, defaultErr case storage.ErrUnauthorized, storage.ErrInvalidSignature: return http.StatusUnauthorized, defaultErr diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 74357d2c1e..6ac066368b 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -29,6 +29,7 @@ import ( "strings" "testing" + "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" @@ -37,11 +38,9 @@ import ( ) func init() { - verbose := flag.Bool("v", false, "verbose") + loglevel := flag.Int("loglevel", 2, "loglevel") flag.Parse() - if *verbose { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) - } + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) } type resourceResponse struct { @@ -55,10 +54,8 @@ func TestBzzResource(t *testing.T) { defer srv.Close() // our mutable resource "name" - keybytes := []byte("foo") - srv.Hasher.Reset() - srv.Hasher.Write([]byte(fmt.Sprintf("%x", keybytes))) - keybyteshash := fmt.Sprintf("%x", srv.Hasher.Sum(nil)) + keybytes := "foo" + keybyteshash := ens.EnsNode(keybytes) // data of update 1 databytes := make([]byte, 666) @@ -68,7 +65,7 @@ func TestBzzResource(t *testing.T) { } // creates resource and sets update 1 - url := fmt.Sprintf("%s/bzz-resource:/%x/13", srv.URL, keybytes) + url := fmt.Sprintf("%s/bzz-resource:/%s/13", srv.URL, []byte(keybytes)) resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes)) if err != nil { t.Fatal(err) @@ -86,8 +83,8 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatalf("data %s could not be unmarshaled: %v", b, err) } - if rsrcResp.Update.Hex() != keybyteshash { - t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", keybyteshash, rsrcResp.Resource) + if !bytes.Equal(rsrcResp.Update, keybyteshash.Bytes()) { + t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", keybyteshash.Hex(), rsrcResp.Update.Hex()) } // get manifest @@ -131,8 +128,16 @@ func TestBzzResource(t *testing.T) { t.Fatal(err) } + // get non-existent name + url = fmt.Sprintf("%s/bzz-resource:/bar", srv.URL) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + // get latest update (1.1) through resource directly - url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -150,7 +155,7 @@ func TestBzzResource(t *testing.T) { } // update 2 - url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes) data := []byte("foo") resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data)) if err != nil { @@ -162,7 +167,7 @@ func TestBzzResource(t *testing.T) { } // get latest update (1.2) through resource directly - url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -180,7 +185,7 @@ func TestBzzResource(t *testing.T) { } // get latest update (1.2) with specified period - url = fmt.Sprintf("%s/bzz-resource:/%x/1", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%s/1", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) @@ -198,7 +203,7 @@ func TestBzzResource(t *testing.T) { } // get first update (1.1) with specified period and version - url = fmt.Sprintf("%s/bzz-resource:/%x/1/1", srv.URL, keybytes) + url = fmt.Sprintf("%s/bzz-resource:/%s/1/1", srv.URL, keybytes) resp, err = http.Get(url) if err != nil { t.Fatal(err) diff --git a/swarm/fuse/swarmfs_test.go b/swarm/fuse/swarmfs_test.go index b385a72587..73a03ad8c4 100644 --- a/swarm/fuse/swarmfs_test.go +++ b/swarm/fuse/swarmfs_test.go @@ -21,6 +21,7 @@ package fuse import ( "bytes" "crypto/rand" + "flag" "io" "io/ioutil" "os" @@ -29,8 +30,23 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" + + "github.com/ethereum/go-ethereum/log" + + colorable "github.com/mattn/go-colorable" ) +var ( + loglevel = flag.Int("loglevel", 4, "verbosity of logs") + rawlog = flag.Bool("rawlog", false, "turn off terminal formatting in logs") +) + +func init() { + flag.Parse() + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(!*rawlog)))) +} + type fileInfo struct { perm uint64 uid int diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index dc2ae5096b..ef6a12bfbb 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -133,7 +133,11 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora os.RemoveAll(datadir) } - localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + params := storage.NewDefaultLocalStoreParams() + params.Init(datadir) + params.BaseKey = addr.Over() + + localStore, err := storage.NewTestLocalStoreForAddr(params) if err != nil { return nil, nil, nil, removeDataDir, err } diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 9f0109fefd..21a2609485 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -138,7 +138,7 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e if created { if err := d.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil { log.Warn("unable to forward chunk request", "peer", sp.ID(), "key", chunk.Key, "err", err) - chunk.SetErrored(true) + chunk.SetErrored(storage.ErrChunkForward) return nil } } @@ -149,10 +149,10 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e select { case <-chunk.ReqC: case <-t.C: - chunk.SetErrored(true) + chunk.SetErrored(storage.ErrChunkTimeout) return } - chunk.SetErrored(false) + chunk.SetErrored(nil) if req.SkipCheck { err := sp.Deliver(chunk, s.priority) @@ -209,6 +209,12 @@ R: chunk.SData = req.SData d.db.Put(chunk) chunk.WaitToStore() + err = chunk.GetErrored() + if err != nil { + if err == storage.ErrChunkInvalid { + req.peer.Drop(err) + } + } close(chunk.ReqC) } } diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 2df870cc77..08f3d500c6 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -662,7 +662,10 @@ func createTestLocalStorageForId(id discover.NodeID, addr *network.BzzAddr) (sto } datadirs[id] = datadir var store storage.ChunkStore - store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + params := storage.NewDefaultLocalStoreParams() + params.ChunkDbPath = datadir + params.BaseKey = addr.Over() + store, err = storage.NewTestLocalStoreForAddr(params) if err != nil { return nil, err } diff --git a/swarm/network/stream/testing/testing.go b/swarm/network/stream/testing/testing.go index 29d8026a22..190d584d12 100644 --- a/swarm/network/stream/testing/testing.go +++ b/swarm/network/stream/testing/testing.go @@ -55,7 +55,10 @@ func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) { break } var store storage.ChunkStore - store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over()) + params := storage.NewDefaultLocalStoreParams() + params.Init(datadir) + params.BaseKey = addr.Over() + store, err = storage.NewTestLocalStoreForAddr(params) if err != nil { break } diff --git a/swarm/storage/dpa.go b/swarm/storage/dpa.go index 9e7b6e9460..0a5bd95c6e 100644 --- a/swarm/storage/dpa.go +++ b/swarm/storage/dpa.go @@ -17,7 +17,6 @@ package storage import ( - "errors" "io" "time" ) @@ -41,8 +40,6 @@ const ( ) var ( - ErrChunkNotFound = errors.New("chunk not found") - ErrFetching = errors.New("chunk still fetching") // timeout interval before retrieval is timed out searchTimeout = 3 * time.Second ) @@ -64,18 +61,14 @@ func NewDPAParams() *DPAParams { // for testing locally func NewLocalDPA(datadir string, basekey []byte) (*DPA, error) { - - hash := MakeHashFunc("SHA3") - - dbStore, err := NewLDBStore(datadir, hash, defaultLDBCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + params := NewDefaultLocalStoreParams() + params.Init(datadir) + localStore, err := NewLocalStore(params, nil) if err != nil { return nil, err } - - return NewDPA(&LocalStore{ - memStore: NewMemStore(dbStore, defaultCacheCapacity, defaultChunkRequestsCacheCapacity), - DbStore: dbStore, - }, NewDPAParams()), nil + localStore.Validators = append(localStore.Validators, NewContentAddressValidator(MakeHashFunc(SHA3Hash))) + return NewDPA(localStore, NewDPAParams()), nil } func NewDPA(store ChunkStore, params *DPAParams) *DPA { diff --git a/swarm/storage/dpa_test.go b/swarm/storage/dpa_test.go index 84ce1c1b19..245c4430a5 100644 --- a/swarm/storage/dpa_test.go +++ b/swarm/storage/dpa_test.go @@ -32,14 +32,14 @@ func TestDPArandom(t *testing.T) { } func testDpaRandom(toEncrypt bool, t *testing.T) { - tdb, err := newTestDbStore(false) + tdb, err := newTestDbStore(false, false) if err != nil { t.Fatalf("init dbStore failed: %v", err) } defer tdb.close() db := tdb.LDBStore db.setCapacity(50000) - memStore := NewMemStore(db, defaultCacheCapacity, defaultChunkRequestsCacheCapacity) + memStore := NewMemStore(NewDefaultStoreParams(), db) localStore := &LocalStore{ memStore: memStore, DbStore: db, @@ -71,7 +71,7 @@ func testDpaRandom(toEncrypt bool, t *testing.T) { } ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666) ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666) - localStore.memStore = NewMemStore(db, defaultCacheCapacity, defaultChunkRequestsCacheCapacity) + localStore.memStore = NewMemStore(NewDefaultStoreParams(), db) resultReader, isEncrypted = dpa.Retrieve(key) if isEncrypted != toEncrypt { t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted) @@ -97,13 +97,13 @@ func TestDPA_capacity(t *testing.T) { } func testDPA_capacity(toEncrypt bool, t *testing.T) { - tdb, err := newTestDbStore(false) + tdb, err := newTestDbStore(false, false) if err != nil { t.Fatalf("init dbStore failed: %v", err) } defer tdb.close() db := tdb.LDBStore - memStore := NewMemStore(db, 0, 0) + memStore := NewMemStore(NewDefaultStoreParams(), db) localStore := &LocalStore{ memStore: memStore, DbStore: db, diff --git a/swarm/storage/error.go b/swarm/storage/error.go index 9a8676c8a3..4f80626233 100644 --- a/swarm/storage/error.go +++ b/swarm/storage/error.go @@ -1,7 +1,12 @@ package storage +import ( + "errors" +) + const ( - ErrNotFound = iota + ErrInit = iota + ErrNotFound ErrIO ErrUnauthorized ErrInvalidValue @@ -12,3 +17,12 @@ const ( ErrPeriodDepth ErrCnt ) + +var ( + ErrChunkNotFound = errors.New("chunk not found") + ErrFetching = errors.New("chunk still fetching") + ErrChunkInvalid = errors.New("invalid chunk") + ErrChunkForward = errors.New("cannot forward") + ErrChunkUnavailable = errors.New("chunk unavailable") + ErrChunkTimeout = errors.New("timeout") +) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 5ad2d71a3f..551943c9c0 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -70,6 +70,21 @@ type gcItem struct { po uint8 } +type LDBStoreParams struct { + *StoreParams + Path string + Po func(Key) uint8 +} + +// NewLDBStoreParams constructs LDBStoreParams with the specified values. +func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams { + return &LDBStoreParams{ + StoreParams: storeparams, + Path: path, + Po: func(k Key) (ret uint8) { return uint8(Proximity(storeparams.BaseKey[:], k[:])) }, + } +} + type LDBStore struct { db *LDBDatabase @@ -87,7 +102,6 @@ type LDBStore struct { batchesC chan struct{} batch *leveldb.Batch lock sync.RWMutex - trusted bool // if hash integity check is to be performed (for testing only) // Functions encodeDataFunc is used to bypass // the default functionality of DbStore with @@ -102,9 +116,9 @@ type LDBStore struct { // TODO: Instead of passing the distance function, just pass the address from which distances are calculated // to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing // a function different from the one that is actually used. -func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8) (s *LDBStore, err error) { +func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) { s = new(LDBStore) - s.hashfunc = hash + s.hashfunc = params.Hash s.batchC = make(chan bool) s.batchesC = make(chan struct{}, 1) @@ -113,13 +127,13 @@ func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) ui // associate encodeData with default functionality s.encodeDataFunc = encodeData - s.db, err = NewLDBDatabase(path) + s.db, err = NewLDBDatabase(params.Path) if err != nil { return nil, err } - s.po = po - s.setCapacity(capacity) + s.po = params.Po + s.setCapacity(params.DbCapacity) s.bucketCnt = make([]uint64, 0x100) for i := 0; i < 0x100; i++ { @@ -143,15 +157,11 @@ func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) ui return s, nil } -func (self *LDBStore) SetTrusted() { - self.trusted = true -} - // NewMockDbStore creates a new instance of DbStore with // mockStore set to a provided value. If mockStore argument is nil, // this function behaves exactly as NewDbStore. -func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8, mockStore *mock.NodeStore) (s *LDBStore, err error) { - s, err = NewLDBStore(path, hash, capacity, po) +func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error) { + s, err = NewLDBStore(params) if err != nil { return nil, err } @@ -344,7 +354,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { continue } - key, err := hex.DecodeString(hdr.Name) + keybytes, err := hex.DecodeString(hdr.Name) if err != nil { log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err) continue @@ -354,6 +364,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { if err != nil { return count, err } + key := Key(keybytes) chunk := NewChunk(key, nil) chunk.SData = data[32:] s.Put(chunk) @@ -631,19 +642,6 @@ func (s *LDBStore) get(key Key) (chunk *Chunk, err error) { } } - if !s.trusted { - data_mod := data[32:] - hasher := s.hashfunc() - hasher.Write(data_mod) - hash := hasher.Sum(nil) - - if !bytes.Equal(hash, key) { - log.Error("apparent key/hash mismatch", "hash", hash, "key", key[:]) - s.delete(indx.Idx, getIndexKey(key), s.po(key)) - log.Error("invalid chunk in database.") - } - } - chunk = NewChunk(key, nil) chunk.markAsStored() decodeData(data, chunk) diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 5c6585af2c..3702bef0fa 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -37,21 +37,25 @@ type testDbStore struct { dir string } -func newTestDbStore(mock bool) (*testDbStore, error) { +func newTestDbStore(mock bool, trusted bool) (*testDbStore, error) { dir, err := ioutil.TempDir("", "bzz-storage-test") if err != nil { return nil, err } var db *LDBStore + storeparams := NewDefaultStoreParams() + params := NewLDBStoreParams(storeparams, dir) + params.Po = testPoFunc + if mock { globalStore := mem.NewGlobalStore() addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed") mockStore := globalStore.NewNodeStore(addr) - db, err = NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultLDBCapacity, testPoFunc, mockStore) + db, err = NewMockDbStore(params, mockStore) } else { - db, err = NewLDBStore(dir, MakeHashFunc(SHA3Hash), defaultLDBCapacity, testPoFunc) + db, err = NewLDBStore(params) } return &testDbStore{db, dir}, err @@ -71,17 +75,16 @@ func (db *testDbStore) close() { } func testDbStoreRandom(n int, processors int, chunksize int, mock bool, t *testing.T) { - db, err := newTestDbStore(mock) + db, err := newTestDbStore(mock, true) if err != nil { t.Fatalf("init dbStore failed: %v", err) } defer db.close() - db.trusted = true testStoreRandom(db, processors, n, chunksize, t) } func testDbStoreCorrect(n int, processors int, chunksize int, mock bool, t *testing.T) { - db, err := newTestDbStore(mock) + db, err := newTestDbStore(mock, false) if err != nil { t.Fatalf("init dbStore failed: %v", err) } @@ -138,7 +141,7 @@ func TestMockDbStoreCorrect_8_5k(t *testing.T) { } func testDbStoreNotFound(t *testing.T, mock bool) { - db, err := newTestDbStore(mock) + db, err := newTestDbStore(mock, false) if err != nil { t.Fatalf("init dbStore failed: %v", err) } @@ -169,7 +172,7 @@ func testIterator(t *testing.T, mock bool) { chunks = append(chunks, NewChunk(nil, nil)) } - db, err := newTestDbStore(mock) + db, err := newTestDbStore(mock, false) if err != nil { t.Fatalf("init dbStore failed: %v", err) } @@ -224,22 +227,20 @@ func TestMockIterator(t *testing.T) { } func benchmarkDbStorePut(n int, processors int, chunksize int, mock bool, b *testing.B) { - db, err := newTestDbStore(mock) + db, err := newTestDbStore(mock, true) if err != nil { b.Fatalf("init dbStore failed: %v", err) } defer db.close() - db.trusted = true benchmarkStorePut(db, processors, n, chunksize, b) } func benchmarkDbStoreGet(n int, processors int, chunksize int, mock bool, b *testing.B) { - db, err := newTestDbStore(mock) + db, err := newTestDbStore(mock, true) if err != nil { b.Fatalf("init dbStore failed: %v", err) } defer db.close() - db.trusted = true benchmarkStoreGet(db, processors, n, chunksize, b) } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 53a6e9f90b..4ab87c3e93 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -19,6 +19,7 @@ package storage import ( "encoding/binary" "fmt" + "path/filepath" "sync" "github.com/ethereum/go-ethereum/log" @@ -30,57 +31,75 @@ var ( dbStorePutCounter = metrics.NewRegisteredCounter("storage.db.dbstore.put.count", nil) ) -type StoreParams struct { - ChunkDbPath string - DbCapacity uint64 - CacheCapacity uint - Radius int +type LocalStoreParams struct { + *StoreParams + ChunkDbPath string + Validators []ChunkValidator `toml:"-"` } -//create params with default values -func NewDefaultStoreParams() (self *StoreParams) { - return &StoreParams{ - DbCapacity: defaultLDBCapacity, - CacheCapacity: defaultCacheCapacity, +func NewDefaultLocalStoreParams() *LocalStoreParams { + return &LocalStoreParams{ + StoreParams: NewDefaultStoreParams(), + } +} + +//this can only finally be set after all config options (file, cmd line, env vars) +//have been evaluated +func (self *LocalStoreParams) Init(path string) { + if self.ChunkDbPath == "" { + self.ChunkDbPath = filepath.Join(path, "chunks") } } // LocalStore is a combination of inmemory db over a disk persisted db // implements a Get/Put with fallback (caching) logic using any 2 ChunkStores type LocalStore struct { - memStore *MemStore - DbStore *LDBStore - mu sync.Mutex + Validators []ChunkValidator + memStore *MemStore + DbStore *LDBStore + mu sync.Mutex } // This constructor uses MemStore and DbStore as components -func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte, mockStore *mock.NodeStore) (*LocalStore, error) { - dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }, mockStore) +func NewLocalStore(params *LocalStoreParams, mockStore *mock.NodeStore) (*LocalStore, error) { + ldbparams := NewLDBStoreParams(params.StoreParams, params.ChunkDbPath) + dbStore, err := NewMockDbStore(ldbparams, mockStore) if err != nil { return nil, err } return &LocalStore{ - memStore: NewMemStore(dbStore, params.CacheCapacity, defaultChunkRequestsCacheCapacity), - DbStore: dbStore, + memStore: NewMemStore(params.StoreParams, dbStore), + DbStore: dbStore, + Validators: params.Validators, }, nil } -func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) { - hasher := MakeHashFunc("SHA3") - dbStore, err := NewLDBStore(path, hasher, defaultLDBCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) +func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) { + ldbparams := NewLDBStoreParams(params.StoreParams, params.ChunkDbPath) + dbStore, err := NewLDBStore(ldbparams) if err != nil { return nil, err } localStore := &LocalStore{ - memStore: NewMemStore(dbStore, defaultCacheCapacity, defaultChunkRequestsCacheCapacity), - DbStore: dbStore, + memStore: NewMemStore(params.StoreParams, dbStore), + DbStore: dbStore, + Validators: params.Validators, } return localStore, nil } -// LocalStore is itself a chunk store -// unsafe, in that the data is not integrity checked func (self *LocalStore) Put(chunk *Chunk) { + valid := true + for _, v := range self.Validators { + if valid = v.Validate(chunk.Key, chunk.SData); valid { + break + } + } + if !valid { + chunk.SetErrored(ErrChunkInvalid) + chunk.dbStoredC <- false + return + } log.Trace("localstore.put", "key", chunk.Key) self.mu.Lock() defer self.mu.Unlock() @@ -139,11 +158,11 @@ func (self *LocalStore) GetOrCreateRequest(key Key) (chunk *Chunk, created bool) var err error chunk, err = self.get(key) - if err == nil && !chunk.GetErrored() { + if err == nil && chunk.GetErrored() == nil { log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", key)) return chunk, false } - if err == ErrFetching && !chunk.GetErrored() { + if err == ErrFetching && chunk.GetErrored() == nil { log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request %v", key, chunk.ReqC)) return chunk, false } diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go new file mode 100644 index 0000000000..37fd3d47d9 --- /dev/null +++ b/swarm/storage/localstore_test.go @@ -0,0 +1,153 @@ +package storage + +import ( + "io/ioutil" + "os" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/contracts/ens" +) + +var ( + hashfunc = MakeHashFunc("SHA3") +) + +// convenience generator for unique chunks +type randomChunkGenerator struct { + data []byte + hasher SwarmHash +} + +func newRandomChunkGenerator(stem []byte) *randomChunkGenerator { + gen := &randomChunkGenerator{ + data: make([]byte, 8), + hasher: hashfunc(), + } + gen.data = append(gen.data, stem...) + return gen +} + +func (self *randomChunkGenerator) newChunk() *Chunk { + self.hasher.Reset() + self.hasher.Write(self.data) + chunk := NewChunk(self.hasher.Sum(nil), nil) + chunk.SData = make([]byte, len(self.data)) + copy(chunk.SData, self.data) + self.data[0]++ + return chunk +} + +// put to localstore and wait for stored channel +// does not check delivery error state +func putChunks(store *LocalStore, chunks ...*Chunk) { + wg := sync.WaitGroup{} + wg.Add(len(chunks)) + go func() { + for _, c := range chunks { + <-c.dbStoredC + wg.Done() + } + }() + for _, c := range chunks { + go store.Put(c) + } + wg.Wait() +} + +// tests that the content address validator correctly checks the data +// tests that resource update chunks are passed through content address validator +// the test checking the resouce update validator internal correctness is found in resource_test.go +func TestValidator(t *testing.T) { + bogusData := []byte("00000000bar") + + // set up localstore + datadir, err := ioutil.TempDir("", "storage-testvalidator") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(datadir) + + params := NewDefaultLocalStoreParams() + params.Init(datadir) + store, err := NewLocalStore(params, nil) + if err != nil { + t.Fatal(err) + } + + // check puts with no validators, both succeed + gen := newRandomChunkGenerator([]byte("foo")) + goodChunk := gen.newChunk() + badChunk := gen.newChunk() + badChunk.SData = bogusData + + putChunks(store, goodChunk, badChunk) + if err := goodChunk.GetErrored(); err != nil { + t.Fatalf("expected no error on good content address chunk in spite of no validation, but got: %s", err) + } + if err := badChunk.GetErrored(); err != nil { + t.Fatalf("expected no error on bad content address chunk in spite of no validation, but got: %s", err) + } + + // add content address validator and check puts + // bad should fail, good should pass + store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc)) + goodChunk = gen.newChunk() + badChunk = gen.newChunk() + badChunk.SData = bogusData + + putChunks(store, goodChunk, badChunk) + if err := goodChunk.GetErrored(); err != nil { + t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err) + } + if err := badChunk.GetErrored(); err == nil { + t.Fatal("expected error on bad content address chunk with content address validator only, but got nil") + } + + // append resource validator to validators and check puts + // bad should fail, good should pass, resource should pass + rhParams := &ResourceHandlerParams{} + rh, err := NewResourceHandler(rhParams) + if err != nil { + t.Fatal(err) + } + store.Validators = append(store.Validators, rh) + + goodChunk = gen.newChunk() + key := rh.resourceHash(42, 1, ens.EnsNode("xyzzy.eth")) + data := []byte("bar") + uglyChunk := newUpdateChunk(key, nil, 42, 1, "xyzzy.eth", data, len(data)) + + putChunks(store, goodChunk, badChunk, uglyChunk) + if err := goodChunk.GetErrored(); err != nil { + t.Fatalf("expected no error on good content address chunk with both validators, but got: %s", err) + } + if err := badChunk.GetErrored(); err == nil { + t.Fatal("expected error on bad chunk address with both validators, but got nil") + } + if err := uglyChunk.GetErrored(); err != nil { + t.Fatalf("expected no error on resource update chunk with both validators, but got: %s", err) + } + + // (redundant check) + // use only resource validator, and check puts + // bad should fail, good should fail, resource should pass + store.Validators[0] = store.Validators[1] + store.Validators = store.Validators[:1] + + goodChunk = gen.newChunk() + key = rh.resourceHash(42, 2, ens.EnsNode("xyzzy.eth")) + data = []byte("baz") + uglyChunk = newUpdateChunk(key, nil, 42, 2, "xyzzy.eth", data, len(data)) + + putChunks(store, goodChunk, badChunk, uglyChunk) + if goodChunk.GetErrored() == nil { + t.Fatal("expected error on good content address chunk with resource validator only, but got nil") + } + if badChunk.GetErrored() == nil { + t.Fatal("expected error on bad content address chunk with resource validator only, but got nil") + } + if err := uglyChunk.GetErrored(); err != nil { + t.Fatalf("expected no error on resource update chunk with resource validator only, but got: %s", err) + } +} diff --git a/swarm/storage/memstore_lrucache.go b/swarm/storage/memstore_lrucache.go index 0c851c0fe8..3bbf923d79 100644 --- a/swarm/storage/memstore_lrucache.go +++ b/swarm/storage/memstore_lrucache.go @@ -33,8 +33,8 @@ type MemStore struct { disabled bool } -func NewMemStore(_ *LDBStore, cacheCapacity uint, requestsCapacity uint) (m *MemStore) { - if cacheCapacity == 0 { +func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore) { + if params.CacheCapacity == 0 { return &MemStore{ disabled: true, } @@ -44,12 +44,12 @@ func NewMemStore(_ *LDBStore, cacheCapacity uint, requestsCapacity uint) (m *Mem v := value.(*Chunk) <-v.dbStoredC } - c, err := lru.NewWithEvict(int(cacheCapacity), onEvicted) + c, err := lru.NewWithEvict(int(params.CacheCapacity), onEvicted) if err != nil { panic(err) } - r, err := lru.New(int(requestsCapacity)) + r, err := lru.New(int(params.ChunkRequestsCacheCapacity)) if err != nil { panic(err) } @@ -98,8 +98,7 @@ func (m *MemStore) Put(c *Chunk) { if c.ReqC != nil { select { case <-c.ReqC: - ok := c.GetErrored() - if !ok { + if c.GetErrored() != nil { m.requests.Remove(string(c.Key)) return } @@ -120,7 +119,24 @@ func (m *MemStore) setCapacity(n int) { if n <= 0 { m.disabled = true } else { - m = NewMemStore(nil, uint(n), defaultChunkRequestsCacheCapacity) + onEvicted := func(key interface{}, value interface{}) { + v := value.(*Chunk) + <-v.dbStoredC + } + c, err := lru.NewWithEvict(n, onEvicted) + if err != nil { + panic(err) + } + + r, err := lru.New(defaultChunkRequestsCacheCapacity) + if err != nil { + panic(err) + } + + m = &MemStore{ + cache: c, + requests: r, + } } } diff --git a/swarm/storage/memstore_lrucache_test.go b/swarm/storage/memstore_lrucache_test.go index 0872f240f3..8765bed40e 100644 --- a/swarm/storage/memstore_lrucache_test.go +++ b/swarm/storage/memstore_lrucache_test.go @@ -34,7 +34,8 @@ func newLDBStore(t *testing.T) (*LDBStore, func()) { } log.Trace("memstore.tempdir", "dir", dir) - db, err := NewLDBStore(dir, MakeHashFunc(SHA3Hash), defaultLDBCapacity, testPoFunc) + ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir) + db, err := NewLDBStore(ldbparams) if err != nil { t.Fatal(err) } @@ -57,7 +58,7 @@ func TestMemStoreAndLDBStore(t *testing.T) { cacheCap := 200 requestsCap := 200 - memStore := NewMemStore(ldb, uint(cacheCap), uint(requestsCap)) + memStore := NewMemStore(NewStoreParams(4000, 200, 200, nil, nil), nil) tests := []struct { n int // number of chunks to push to memStore diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 9146b298c3..f2e2f81a3c 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -19,7 +19,8 @@ package storage import "testing" func newTestMemStore() *MemStore { - return NewMemStore(nil, defaultCacheCapacity, defaultChunkRequestsCacheCapacity) + storeparams := NewDefaultStoreParams() + return NewMemStore(storeparams, nil) } func testMemStoreRandom(n int, processors int, chunksize int, t *testing.T) { diff --git a/swarm/storage/memstore_tree.god b/swarm/storage/memstore_tree.god index 9bb793151b..529b9dfa24 100644 --- a/swarm/storage/memstore_tree.god +++ b/swarm/storage/memstore_tree.god @@ -62,7 +62,9 @@ a hash prefix subtree containing subtrees or one storage entry (but never both) (access[] is a binary tree inside the multi-bit leveled hash tree) */ -func NewMemStore(d *LDBStore, capacity uint) (m *MemStore) { +func NewMemStore(params *StoreParams, d *LDBStore) (m *MemStore) { + + capacity := params.CacheCapacity m = &MemStore{} m.memtree = newMemTree(memTreeFLW, nil, 0) m.ldbStore = d diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 4f1b6021a8..bb433221e7 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -17,7 +17,6 @@ package storage import ( - "path/filepath" "time" ) @@ -57,7 +56,7 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { err := self.retrieve(chunk) if err != nil { // mark chunk request as failed so that we can retry it later - chunk.SetErrored(true) + chunk.SetErrored(ErrChunkUnavailable) return nil, err } } @@ -69,22 +68,14 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { select { case <-t.C: // mark chunk request as failed so that we can retry - chunk.SetErrored(true) + chunk.SetErrored(ErrChunkNotFound) return nil, ErrChunkNotFound case <-chunk.ReqC: } - chunk.SetErrored(false) + chunk.SetErrored(nil) return chunk, nil } -//this can only finally be set after all config options (file, cmd line, env vars) -//have been evaluated -func (self *StoreParams) Init(path string) { - if self.ChunkDbPath == "" { - self.ChunkDbPath = filepath.Join(path, "chunks") - } -} - // Put is the entrypoint for local store requests coming from storeLoop func (self *NetStore) Put(chunk *Chunk) { self.localStore.Put(chunk) diff --git a/swarm/storage/netstore_test.go b/swarm/storage/netstore_test.go index ff2ee96958..acee632046 100644 --- a/swarm/storage/netstore_test.go +++ b/swarm/storage/netstore_test.go @@ -80,8 +80,10 @@ func TestNetstoreFailedRequest(t *testing.T) { if err != nil { t.Fatal(err) } - - localStore, err := NewTestLocalStoreForAddr(datadir, addr.Over()) + params := NewDefaultLocalStoreParams() + params.Init(datadir) + params.BaseKey = addr.Over() + localStore, err := NewTestLocalStoreForAddr(params) if err != nil { t.Fatal(err) } diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 1b06fba39e..2bc0884bf0 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -1,6 +1,7 @@ package storage import ( + "bytes" "context" "encoding/binary" "errors" @@ -13,6 +14,7 @@ import ( "golang.org/x/net/idna" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/contracts/ens" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -20,11 +22,12 @@ import ( const ( signatureLength = 65 - indexSize = 16 + indexSize = 18 DbDirName = "resource" chunkSize = 4096 // temporary until we implement DPA in the resourcehandler defaultStoreTimeout = 4000 * time.Millisecond hasherCount = 8 + resourceHash = SHA3Hash ) type ResourceError struct { @@ -61,10 +64,6 @@ type ResourceLookupParams struct { Max uint32 } -type SignFunc func(common.Hash) (Signature, error) - -type nameHashFunc func(string) common.Hash - // Encapsulates an specific resource update. When synced it contains the most recent // version of the resource update data. type resource struct { @@ -88,15 +87,6 @@ func (self *resource) NameHash() common.Hash { return self.nameHash } -// Implement to activate validation of resource updates -// Specifically signing data and verification of signatures -type ResourceValidator interface { - hashSize() int - checkAccess(string, common.Address) (bool, error) - NameHash(string) common.Hash // nameHashFunc - sign(common.Hash) (Signature, error) // SignFunc -} - type headerGetter interface { HeaderByNumber(context.Context, string, *big.Int) (*types.Header, error) } @@ -146,90 +136,123 @@ type headerGetter interface { // headerlength|period|version|identifier|data // // if a validator is active, the chunk data is: -// sign(resourcedata)|resourcedata +// resourcedata|sign(resourcedata) // otherwise, the chunk data is the same as the resourcedata // // headerlength is a 16 bit value containing the byte length of period|version|name -// period and version are both 32 bit values. name can have arbitrary length -// -// NOTE: the following is yet to be implemented -// The resource update chunks will be stored in the swarm, but receive special -// treatment as their keys do not validate as hashes of their data. They are also -// stored using a separate store, and forwarding/syncing protocols carry per-chunk -// flags to tell whether the chunk can be validated or not; if not it is to be -// treated as a resource update chunk. // // TODO: Include modtime in chunk data + signature type ResourceHandler struct { - ChunkStore - validator ResourceValidator + chunkStore ChunkStore + HashSize int + signer ResourceSigner ethClient headerGetter + ensClient *ens.ENS resources map[string]*resource hashPool sync.Pool resourceLock sync.RWMutex - nameHash nameHashFunc storeTimeout time.Duration queryMaxPeriods *ResourceLookupParams } type ResourceHandlerParams struct { - Validator ResourceValidator QueryMaxPeriods *ResourceLookupParams + Signer ResourceSigner + EthClient headerGetter + EnsClient *ens.ENS } // Create or open resource update chunk store -func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient headerGetter, params *ResourceHandlerParams) (*ResourceHandler, error) { +func NewResourceHandler(params *ResourceHandlerParams) (*ResourceHandler, error) { if params.QueryMaxPeriods == nil { params.QueryMaxPeriods = &ResourceLookupParams{ Limit: false, } } rh := &ResourceHandler{ - ChunkStore: chunkStore, - ethClient: ethClient, + ethClient: params.EthClient, + ensClient: params.EnsClient, resources: make(map[string]*resource), - validator: params.Validator, storeTimeout: defaultStoreTimeout, + signer: params.Signer, hashPool: sync.Pool{ New: func() interface{} { - return MakeHashFunc(SHA3Hash)() + return MakeHashFunc(resourceHash)() }, }, queryMaxPeriods: params.QueryMaxPeriods, } - if rh.validator != nil { - rh.nameHash = rh.validator.NameHash - } else { - rh.nameHash = func(name string) common.Hash { - hasher := rh.hashPool.Get().(SwarmHash) - defer rh.hashPool.Put(hasher) - hasher.Reset() - hasher.Write([]byte(name)) - hashval := common.BytesToHash(hasher.Sum(nil)) - log.Debug("generic namehasher", "name", name, "hash", hashval) - return hashval - } - } - for i := 0; i < hasherCount; i++ { - hashfunc := MakeHashFunc(SHA3Hash)() + hashfunc := MakeHashFunc(resourceHash)() + if rh.HashSize == 0 { + rh.HashSize = hashfunc.Size() + } rh.hashPool.Put(hashfunc) } return rh, nil } -func (self *ResourceHandler) IsValidated() bool { - return self.validator == nil +// Sets the store backend for resource updates +func (self *ResourceHandler) SetStore(store ChunkStore) { + self.chunkStore = store } -func (self *ResourceHandler) HashSize() int { - return self.validator.hashSize() +// Chunk Validation method (matches ChunkValidatorFunc signature) +// +// If resource update, owner is checked against ENS record of resource name inferred from chunk data +// If parsed signature is nil, validates automatically +// If not resource update, it validates are root chunk if length is indexSize and first two bytes are 0 +func (self *ResourceHandler) Validate(key Key, data []byte) bool { + signature, period, version, name, parseddata, err := self.parseUpdate(data) + if err != nil { + if len(data) == indexSize { + if bytes.Equal(data[:2], []byte{0, 0}) { + return true + } + } + return false + } else if signature == nil { + if !bytes.Equal(self.resourceHash(period, version, ens.EnsNode(name)), key) { + return false + } + return true + } + digest := self.keyDataHash(key, parseddata) + addr, err := getAddressFromDataSig(digest, *signature) + if err != nil { + return false + } + ok, _ := self.checkAccess(name, addr) + return ok +} + +// If no ens client is supplied, resource updates are not validated +func (self *ResourceHandler) IsValidated() bool { + return self.ensClient != nil +} + +// Create the resource update digest used in signatures +func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash { + hasher := self.hashPool.Get().(SwarmHash) + defer self.hashPool.Put(hasher) + hasher.Reset() + hasher.Write(key[:]) + hasher.Write(data) + return common.BytesToHash(hasher.Sum(nil)) +} + +// Checks if current address matches owner address of ENS +func (self *ResourceHandler) checkAccess(name string, address common.Address) (bool, error) { + if self.ensClient == nil { + return true, nil + } + owneraddress, err := self.ensClient.Owner(ens.EnsNode(name)) + return (address == owneraddress), err } // get data from current resource - func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { @@ -278,10 +301,10 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ return nil, NewResourceError(ErrInvalidValue, fmt.Sprintf("Invalid name: '%s'", name)) } - nameHash := self.nameHash(name) + nameHash := ens.EnsNode(name) - if self.validator != nil { - signature, err := self.validator.sign(nameHash) + if self.signer != nil { + signature, err := self.signer.Sign(nameHash) if err != nil { return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Sign fail: %v", err)) } @@ -289,7 +312,7 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ if err != nil { return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Retrieve address from signature fail: %v", err)) } - ok, err := self.validator.checkAccess(name, addr) + ok, err := self.checkAccess(name, addr) if err != nil { return nil, err } else if !ok { @@ -305,15 +328,17 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ // chunk with key equal to namehash points to data of first blockheight + update frequency // from this we know from what blockheight we should look for updates, and how often - chunk := NewChunk(Key(nameHash.Bytes()), nil) + key := Key(nameHash.Bytes()) + chunk := NewChunk(key, nil) chunk.SData = make([]byte, indexSize) + // root block has first two bytes both, which distinguishes from update bytes val := make([]byte, 8) binary.LittleEndian.PutUint64(val, currentblock) - copy(chunk.SData[:8], val) + copy(chunk.SData[2:10], val) binary.LittleEndian.PutUint64(val, frequency) - copy(chunk.SData[8:], val) - self.Put(chunk) + copy(chunk.SData[10:], val) + self.chunkStore.Put(chunk) log.Debug("new resource", "name", name, "key", nameHash, "startBlock", currentblock, "frequency", frequency) rsrc := &resource{ @@ -336,12 +361,8 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ // root chunk. // It is the callers responsibility to make sure that this chunk exists (if the resource // update root data was retrieved externally, it typically doesn't) -// -// If maxPeriod is -1, the default QueryMaxPeriod from ResourceHandlerParams will be used -// if maxPeriod is 0, there will be no limit on period hops -// if maxPeriod > 0, the given value will be the limit of period hops func (self *ResourceHandler) LookupVersionByName(ctx context.Context, name string, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) { - return self.LookupVersion(ctx, self.nameHash(name), name, period, version, refresh, maxLookup) + return self.LookupVersion(ctx, ens.EnsNode(name), name, period, version, refresh, maxLookup) } func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.Hash, name string, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) { @@ -361,7 +382,7 @@ func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common. // // See also (*ResourceHandler).LookupVersion func (self *ResourceHandler) LookupHistoricalByName(ctx context.Context, name string, period uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) { - return self.LookupHistorical(ctx, self.nameHash(name), name, period, refresh, maxLookup) + return self.LookupHistorical(ctx, ens.EnsNode(name), name, period, refresh, maxLookup) } func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash common.Hash, name string, period uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) { @@ -383,7 +404,7 @@ func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash comm // // See also (*ResourceHandler).LookupHistorical func (self *ResourceHandler) LookupLatestByName(ctx context.Context, name string, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) { - return self.LookupLatest(ctx, self.nameHash(name), name, refresh, maxLookup) + return self.LookupLatest(ctx, ens.EnsNode(name), name, refresh, maxLookup) } func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.Hash, name string, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) { @@ -404,6 +425,10 @@ func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.H // base code for public lookup methods func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) { + if self.chunkStore == nil { + return nil, NewResourceError(ErrInit, "Call ResourceHandler.SetStore() before performing lookups") + } + if period == 0 { return nil, NewResourceError(ErrInvalidValue, "period must be >0") } @@ -426,7 +451,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 return nil, NewResourceError(ErrPeriodDepth, fmt.Sprintf("Lookup exceeded max period hops (%d)", maxLookup.Max)) } key := self.resourceHash(period, version, rsrc.nameHash) - chunk, err := self.Get(key) + chunk, err := self.chunkStore.Get(key) if err == nil { if specificversion { return self.updateResourceIndex(rsrc, chunk) @@ -436,7 +461,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 for { newversion := version + 1 key := self.resourceHash(period, newversion, rsrc.nameHash) - newchunk, err := self.Get(key) + newchunk, err := self.chunkStore.Get(key) if err != nil { return self.updateResourceIndex(rsrc, chunk) } @@ -472,7 +497,7 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref rsrc.nameHash = nameHash // get the root info chunk and update the cached value - chunk, err := self.Get(Key(rsrc.nameHash[:])) + chunk, err := self.chunkStore.Get(Key(rsrc.nameHash[:])) if err != nil { return nil, err } @@ -481,8 +506,8 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref if len(chunk.SData) != indexSize { return nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize)) } - rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[:8]) - rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[8:]) + rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[2:10]) + rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[10:]) } else { rsrc.name = self.resources[name].name rsrc.nameHash = self.resources[name].nameHash @@ -501,8 +526,8 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( return nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Update belongs to '%s', but have '%s'", name, *rsrc.name)) } log.Trace("update", "name", *rsrc.name, "rootkey", rsrc.nameHash, "updatekey", chunk.Key, "period", period, "version", version) - // only check signature if validator is present - if self.validator != nil { + // check signature (if signer algorithm is present) + if signature != nil { digest := self.keyDataHash(chunk.Key, data) _, err = getAddressFromDataSig(digest, *signature) if err != nil { @@ -525,11 +550,16 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( // retrieve update metadata from chunk data // mirrors newUpdateChunk() func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, uint32, string, []byte, error) { + // 14 = header + one byte of name + one byte of data + if len(chunkdata) < 14 { + return nil, 0, 0, "", nil, NewResourceError(ErrNothingToReturn, "chunk less than 13 bytes cannot be a resource update chunk") + } cursor := 0 headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) cursor += 2 datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) - if int(headerlength+datalength+4) > len(chunkdata) { + exclsignlength := int(headerlength + datalength + 4) + if exclsignlength > len(chunkdata) || exclsignlength < 14 { return nil, 0, 0, "", nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata))) } var period uint32 @@ -560,10 +590,13 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, // omit signatures if we have no validator var signature *Signature - if self.validator != nil { - cursor += intdatalength - signature = &Signature{} - copy(signature[:], chunkdata[cursor:cursor+signatureLength]) + cursor += intdatalength + if self.signer != nil { + sigdata := chunkdata[cursor : cursor+signatureLength] + if len(sigdata) > 0 { + signature = &Signature{} + copy(signature[:], sigdata) + } } return signature, period, version, name, data, nil @@ -589,8 +622,11 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt func (self *ResourceHandler) update(ctx context.Context, name string, data []byte, multihash bool) (Key, error) { + if self.chunkStore == nil { + return nil, NewResourceError(ErrInit, "Call ResourceHandler.SetStore() before updating") + } var signaturelength int - if self.validator != nil { + if self.signer != nil { signaturelength = signatureLength } @@ -628,10 +664,10 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt key := self.resourceHash(nextperiod, version, rsrc.nameHash) var signature *Signature - if self.validator != nil { + if self.signer != nil { // sign the data hash with the key digest := self.keyDataHash(key, data) - sig, err := self.validator.sign(digest) + sig, err := self.signer.Sign(digest) if err != nil { return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Sign fail: %v", err)) } @@ -642,13 +678,14 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt if err != nil { return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Invalid data/signature: %v", err)) } - - // check if the signer has access to update - ok, err := self.validator.checkAccess(name, addr) - if err != nil { - return nil, NewResourceError(ErrIO, fmt.Sprintf("Access check fail: %v", err)) - } else if !ok { - return nil, NewResourceError(ErrUnauthorized, fmt.Sprintf("Address %x does not have access to update %s", addr, name)) + if self.signer != nil { + // check if the signer has access to update + ok, err := self.checkAccess(name, addr) + if err != nil { + return nil, NewResourceError(ErrIO, fmt.Sprintf("Access check fail: %v", err)) + } else if !ok { + return nil, NewResourceError(ErrUnauthorized, fmt.Sprintf("Address %x does not have access to update %s", addr, name)) + } } } @@ -659,7 +696,7 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt chunk := newUpdateChunk(key, signature, nextperiod, version, name, data, datalength) // send the chunk - self.Put(chunk) + self.chunkStore.Put(chunk) timeout := time.NewTimer(self.storeTimeout) select { case <-chunk.dbStoredC: @@ -679,7 +716,7 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt // Closes the datastore. // Always call this at shutdown to avoid data corruption. func (self *ResourceHandler) Close() { - self.ChunkStore.Close() + self.chunkStore.Close() } func (self *ResourceHandler) getBlock(ctx context.Context, name string) (uint64, error) { @@ -789,52 +826,6 @@ func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32 return chunk } -// \TODO chunkSize is a workaround until the ChunkStore interface exports a method to get the chunk size directly -type resourceChunkStore struct { - localStore ChunkStore - netStore ChunkStore - chunkSize int64 -} - -func NewResourceChunkStore(localStore *LocalStore, request func(*Chunk) error) ChunkStore { - return &resourceChunkStore{ - localStore: localStore, - netStore: NewNetStore(localStore, request), - } -} - -func (r *resourceChunkStore) Get(key Key) (*Chunk, error) { - chunk, err := r.netStore.Get(key) - if err != nil { - return nil, err - } - // if the chunk has to be remotely retrieved, we define a timeout of how long to wait for it before failing. - // sadly due to the nature of swarm, the error will never be conclusive as to whether it was a network issue - // that caused the failure or that the chunk doesn't exist. - if chunk.ReqC == nil { - return chunk, nil - } - t := time.NewTimer(time.Second * 1) - select { - case <-t.C: - log.Trace("Timeout on resource chunk store") - return nil, fmt.Errorf("timeout") - case <-chunk.C: - log.Trace("Received resource update chunk") - } - return chunk, nil -} - -func (r *resourceChunkStore) Put(chunk *Chunk) { - r.netStore.Put(chunk) - chunk.WaitToStore() -} - -func (r *resourceChunkStore) Close() { - r.netStore.Close() - r.localStore.Close() -} - func getNextPeriod(start uint64, current uint64, frequency uint64) uint32 { blockdiff := current - start period := blockdiff / frequency @@ -857,16 +848,6 @@ func isSafeName(name string) bool { return validname == name } -// convenience for creating signature hashes of update data -func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash { - hasher := self.hashPool.Get().(SwarmHash) - defer self.hashPool.Put(hasher) - hasher.Reset() - hasher.Write(key[:]) - hasher.Write(data) - return common.BytesToHash(hasher.Sum(nil)) -} - // if first byte is the start of a multihash this function will try to parse it // if successful it returns the length of multihash data, 0 otherwise func isMultihash(data []byte) int { @@ -892,29 +873,20 @@ func isMultihash(data []byte) int { return cursor + inthashlength } -// TODO: this should not be part of production code, but currently swarm/testutil/http.go needs it -func NewTestResourceHandler(datadir string, ethClient headerGetter, validator ResourceValidator, maxLimit *ResourceLookupParams) (*ResourceHandler, error) { +// TODO: this should not be exposed, but swarm/testutil/http.go needs it +func NewTestResourceHandler(datadir string, params *ResourceHandlerParams) (*ResourceHandler, error) { path := filepath.Join(datadir, DbDirName) - basekey := make([]byte, 32) - hasher := MakeHashFunc(SHA3Hash) - dbStore, err := NewLDBStore(path, hasher, defaultLDBCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) - dbStore.SetTrusted() + rh, err := NewResourceHandler(params) if err != nil { return nil, err } - localStore := &LocalStore{ - memStore: NewMemStore(dbStore, defaultCacheCapacity, defaultChunkRequestsCacheCapacity), - DbStore: dbStore, + localstoreparams := NewDefaultLocalStoreParams() + localstoreparams.Init(path) + localStore, err := NewLocalStore(localstoreparams, nil) + if err != nil { + return nil, err } - resourceChunkStore := NewResourceChunkStore(localStore, nil) - if maxLimit == nil { - maxLimit = &ResourceLookupParams{ - Limit: false, - } - } - params := &ResourceHandlerParams{ - Validator: validator, - QueryMaxPeriods: maxLimit, - } - return NewResourceHandler(hasher, resourceChunkStore, ethClient, params) + localStore.Validators = append(localStore.Validators, rh) + rh.SetStore(localStore) + return rh, nil } diff --git a/swarm/storage/resource_ens.go b/swarm/storage/resource_ens.go deleted file mode 100644 index ad01cf9508..0000000000 --- a/swarm/storage/resource_ens.go +++ /dev/null @@ -1,52 +0,0 @@ -package storage - -import ( - "errors" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/contracts/ens" -) - -type baseValidator struct { - signFunc SignFunc - hashsize int -} - -func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err error) { - if b.signFunc == nil { - return signature, errors.New("No signature function") - } - return b.signFunc(datahash) -} - -func (b *baseValidator) hashSize() int { - return b.hashsize -} - -type OwnerValidator interface { - ValidateOwner(name string, address common.Address) (bool, error) -} - -// ENS validation of mutable resource owners -type ENSValidator struct { - *baseValidator - api OwnerValidator -} - -func NewENSValidator(contractaddress common.Address, ownerValidator OwnerValidator, signFunc SignFunc) *ENSValidator { - return &ENSValidator{ - baseValidator: &baseValidator{ - signFunc: signFunc, - hashsize: common.HashLength, - }, - api: ownerValidator, - } -} - -func (self *ENSValidator) checkAccess(name string, address common.Address) (bool, error) { - return self.api.ValidateOwner(name, address) -} - -func (self *ENSValidator) NameHash(name string) common.Hash { - return ens.EnsNode(name) -} diff --git a/swarm/storage/resource_sign.go b/swarm/storage/resource_sign.go index 840e705ac1..526ae1befd 100644 --- a/swarm/storage/resource_sign.go +++ b/swarm/storage/resource_sign.go @@ -7,14 +7,20 @@ import ( "github.com/ethereum/go-ethereum/crypto" ) -// matches the SignFunc type -func NewGenericResourceSigner(privKey *ecdsa.PrivateKey) SignFunc { - return func(data common.Hash) (signature Signature, err error) { - signaturebytes, err := crypto.Sign(data.Bytes(), privKey) - if err != nil { - return - } - copy(signature[:], signaturebytes) +// Signs resource updates +type ResourceSigner interface { + Sign(common.Hash) (Signature, error) +} + +type GenericResourceSigner struct { + PrivKey *ecdsa.PrivateKey +} + +func (self *GenericResourceSigner) Sign(data common.Hash) (signature Signature, err error) { + signaturebytes, err := crypto.Sign(data.Bytes(), self.PrivKey) + if err != nil { return } + copy(signature[:], signaturebytes) + return } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index 3b4b5d43d0..9fceebbc53 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -3,7 +3,6 @@ package storage import ( "bytes" "context" - "crypto/ecdsa" "crypto/rand" "encoding/binary" "flag" @@ -83,14 +82,14 @@ func TestResourceReverse(t *testing.T) { } // set up rpc and create resourcehandler - rh, _, _, teardownTest, err := setupTest(nil, newTestValidator(signer.signContent)) + rh, _, teardownTest, err := setupTest(nil, nil, signer) if err != nil { t.Fatal(err) } defer teardownTest() // generate a hash for block 4200 version 1 - key := rh.resourceHash(period, version, rh.nameHash(safeName)) + key := rh.resourceHash(period, version, ens.EnsNode(safeName)) // generate some bogus data for the chunk and sign it data := make([]byte, 8) @@ -101,7 +100,7 @@ func TestResourceReverse(t *testing.T) { testHasher.Reset() testHasher.Write(data) digest := rh.keyDataHash(key, data) - sig, err := rh.validator.sign(digest) + sig, err := rh.signer.Sign(digest) if err != nil { t.Fatal(err) } @@ -118,7 +117,7 @@ func TestResourceReverse(t *testing.T) { if err != nil { t.Fatalf("Retrieve address from signature fail: %v", err) } - originaladdress := crypto.PubkeyToAddress(signer.privKey.PublicKey) + originaladdress := crypto.PubkeyToAddress(signer.PrivKey.PublicKey) // check that the metadata retrieved from the chunk matches what we gave it if recoveredaddress != originaladdress { @@ -149,7 +148,7 @@ func TestResourceHandler(t *testing.T) { backend := &fakeBackend{ blocknumber: int64(startBlock), } - rh, datadir, _, teardownTest, err := setupTest(backend, nil) + rh, datadir, teardownTest, err := setupTest(backend, nil, nil) if err != nil { t.Fatal(err) } @@ -164,15 +163,15 @@ func TestResourceHandler(t *testing.T) { } // check that the new resource is stored correctly - namehash := rh.nameHash(safeName) - chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(namehash[:])) + namehash := ens.EnsNode(safeName) + chunk, err := rh.chunkStore.(*LocalStore).memStore.Get(Key(namehash[:])) if err != nil { t.Fatal(err) } else if len(chunk.SData) < 16 { t.Fatalf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData)) } - startblocknumber := binary.LittleEndian.Uint64(chunk.SData[:8]) - chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[8:]) + startblocknumber := binary.LittleEndian.Uint64(chunk.SData[2:10]) + chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[10:]) if startblocknumber != uint64(backend.blocknumber) { t.Fatalf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber) } @@ -219,10 +218,14 @@ func TestResourceHandler(t *testing.T) { // it will match on second iteration startblocknumber + (resourceFrequency * 3) fwdBlocks(int(resourceFrequency*2)-1, backend) - lookupParams := &ResourceLookupParams{ - Limit: false, + rhparams := &ResourceHandlerParams{ + QueryMaxPeriods: &ResourceLookupParams{ + Limit: false, + }, + Signer: nil, + EthClient: rh.ethClient, } - rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil, lookupParams) + rh2, err := NewTestResourceHandler(datadir, rhparams) if err != nil { t.Fatal(err) } @@ -244,7 +247,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, latest version - rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true, lookupParams) + rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true, rh2.queryMaxPeriods) if err != nil { t.Fatal(err) } @@ -255,7 +258,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, specific version - rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true, lookupParams) + rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true, rh2.queryMaxPeriods) if err != nil { t.Fatal(err) } @@ -267,6 +270,65 @@ func TestResourceHandler(t *testing.T) { } +// create ENS enabled resource update, with and without valid owner +func TestResourceENSOwner(t *testing.T) { + + // signer containing private key + signer, err := newTestSigner() + if err != nil { + t.Fatal(err) + } + + // ens address and transact options + addr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey) + transactOpts := bind.NewKeyedTransactor(signer.PrivKey) + + // set up ENS sim + domainparts := strings.Split(safeName, ".") + contractAddr, contractbackend, err := setupENS(addr, transactOpts, domainparts[0], domainparts[1]) + if err != nil { + t.Fatal(err) + } + + ensClient, err := ens.NewENS(transactOpts, contractAddr, contractbackend) + if err != nil { + t.Fatal(err) + } + + // set up rpc and create resourcehandler with ENS sim backend + rh, _, teardownTest, err := setupTest(contractbackend, ensClient, signer) + if err != nil { + t.Fatal(err) + } + defer teardownTest() + + // create new resource when we are owner = ok + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err = rh.NewResource(ctx, safeName, resourceFrequency) + if err != nil { + t.Fatalf("Create resource fail: %v", err) + } + + data := []byte("foo") + // update resource when we are owner = ok + _, err = rh.Update(ctx, safeName, data) + if err != nil { + t.Fatalf("Update resource fail: %v", err) + } + + // update resource when we are not owner = !ok + signertwo, err := newTestSigner() + if err != nil { + t.Fatal(err) + } + rh.signer = signertwo + _, err = rh.Update(ctx, safeName, data) + if err == nil { + t.Fatalf("Expected resource update fail due to owner mismatch") + } +} + func TestResourceMultihash(t *testing.T) { // signer containing private key @@ -274,7 +336,6 @@ func TestResourceMultihash(t *testing.T) { if err != nil { t.Fatal(err) } - validator := newTestValidator(signer.signContent) // make fake backend, set up rpc and create resourcehandler backend := &fakeBackend{ @@ -282,7 +343,7 @@ func TestResourceMultihash(t *testing.T) { } // set up rpc and create resourcehandler - rh, datadir, _, teardownTest, err := setupTest(backend, nil) + rh, datadir, teardownTest, err := setupTest(backend, nil, nil) if err != nil { t.Fatal(err) } @@ -298,7 +359,7 @@ func TestResourceMultihash(t *testing.T) { // we're naïvely assuming keccak256 for swarm hashes // if it ever changes this test should also change - swarmhashbytes := rh.nameHash("foo") + swarmhashbytes := ens.EnsNode("foo") swarmhashmulti, err := multihash.Encode(swarmhashbytes.Bytes(), multihash.KECCAK_256) if err != nil { t.Fatal(err) @@ -352,8 +413,16 @@ func TestResourceMultihash(t *testing.T) { } rh.Close() + rhparams := &ResourceHandlerParams{ + QueryMaxPeriods: &ResourceLookupParams{ + Limit: false, + }, + Signer: signer, + EthClient: rh.ethClient, + EnsClient: rh.ensClient, + } // test with signed data - rh2, err := NewTestResourceHandler(datadir, rh.ethClient, validator, nil) + rh2, err := NewTestResourceHandler(datadir, rhparams) if err != nil { t.Fatal(err) } @@ -394,9 +463,7 @@ func TestResourceMultihash(t *testing.T) { } } -// create ENS enabled resource update, with and without valid owner -func TestResourceENSOwner(t *testing.T) { - +func TestResourceChunkValidator(t *testing.T) { // signer containing private key signer, err := newTestSigner() if err != nil { @@ -404,8 +471,8 @@ func TestResourceENSOwner(t *testing.T) { } // ens address and transact options - addr := crypto.PubkeyToAddress(signer.privKey.PublicKey) - transactOpts := bind.NewKeyedTransactor(signer.privKey) + addr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey) + transactOpts := bind.NewKeyedTransactor(signer.PrivKey) // set up ENS sim domainparts := strings.Split(safeName, ".") @@ -418,10 +485,9 @@ func TestResourceENSOwner(t *testing.T) { if err != nil { t.Fatal(err) } - validator := NewENSValidator(contractAddr, newTestResolver(ensClient), signer.signContent) // set up rpc and create resourcehandler with ENS sim backend - rh, _, _, teardownTest, err := setupTest(contractbackend, validator) + rh, _, teardownTest, err := setupTest(contractbackend, ensClient, signer) if err != nil { t.Fatal(err) } @@ -430,27 +496,21 @@ func TestResourceENSOwner(t *testing.T) { // create new resource when we are owner = ok ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, err = rh.NewResource(ctx, safeName, resourceFrequency) + rsrc, err := rh.NewResource(ctx, safeName, resourceFrequency) if err != nil { t.Fatalf("Create resource fail: %v", err) } data := []byte("foo") - // update resource when we are owner = ok - _, err = rh.Update(ctx, safeName, data) + key := rh.resourceHash(1, 1, rsrc.nameHash) + digest := rh.keyDataHash(key, data) + sig, err := rh.signer.Sign(digest) if err != nil { - t.Fatalf("Update resource fail: %v", err) + t.Fatalf("sign fail: %v", err) } - - // update resource when we are owner = ok - signertwo, err := newTestSigner() - if err != nil { - t.Fatal(err) - } - rh.validator.(*ENSValidator).signFunc = signertwo.signContent - _, err = rh.Update(ctx, safeName, data) - if err == nil { - t.Fatalf("Expected resource update fail due to owner mismatch") + chunk := newUpdateChunk(key, &sig, 1, 1, safeName, data, len(data)) + if !rh.Validate(chunk.Key, chunk.SData) { + t.Fatal("Chunk validator fail") } } @@ -462,7 +522,7 @@ func fwdBlocks(count int, backend *fakeBackend) { } // create rpc and resourcehandler -func setupTest(backend headerGetter, validator ResourceValidator) (rh *ResourceHandler, datadir string, signer *testSigner, teardown func(), err error) { +func setupTest(backend headerGetter, ensBackend *ens.ENS, signer ResourceSigner) (rh *ResourceHandler, datadir string, teardown func(), err error) { var fsClean func() var rpcClean func() @@ -478,14 +538,22 @@ func setupTest(backend headerGetter, validator ResourceValidator) (rh *ResourceH // temp datadir datadir, err = ioutil.TempDir("", "rh") if err != nil { - return nil, "", nil, nil, err + return nil, "", nil, err } fsClean = func() { os.RemoveAll(datadir) } - rh, err = NewTestResourceHandler(datadir, backend, validator, &ResourceLookupParams{Limit: false}) - return rh, datadir, signer, cleanF, nil + rhparams := &ResourceHandlerParams{ + QueryMaxPeriods: &ResourceLookupParams{ + Limit: false, + }, + Signer: signer, + EthClient: backend, + EnsClient: ensBackend, + } + rh, err = NewTestResourceHandler(datadir, rhparams) + return rh, datadir, cleanF, nil } // Set up simulated ENS backend for use with ENSResourceHandler tests @@ -531,55 +599,18 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string, return contractAddress, contractBackend, nil } -// implementation of an external signer to pass to validator -type testSigner struct { - privKey *ecdsa.PrivateKey - hasher SwarmHash - signContent SignFunc -} - -func newTestSigner() (*testSigner, error) { +func newTestSigner() (*GenericResourceSigner, error) { privKey, err := crypto.GenerateKey() if err != nil { return nil, err } - return &testSigner{ - privKey: privKey, - hasher: testHasher, - signContent: NewGenericResourceSigner(privKey), + return &GenericResourceSigner{ + PrivKey: privKey, }, nil } -// Default fallthrough validation of mutable resource ownership -type testValidator struct { - *baseValidator - hashFunc func(string) common.Hash -} - -func newTestValidator(signFunc SignFunc) *testValidator { - return &testValidator{ - baseValidator: &baseValidator{ - signFunc: signFunc, - }, - hashFunc: func(name string) common.Hash { - testHasher.Reset() - testHasher.Write([]byte(name)) - return common.BytesToHash(testHasher.Sum(nil)) - }, - } - -} - -func (self *testValidator) checkAccess(name string, address common.Address) (bool, error) { - return true, nil -} - -func (self *testValidator) NameHash(name string) common.Hash { - return self.hashFunc(name) -} - func getUpdateDirect(rh *ResourceHandler, key Key) ([]byte, error) { - chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(key) + chunk, err := rh.chunkStore.(*LocalStore).memStore.Get(key) if err != nil { return nil, err } @@ -589,18 +620,3 @@ func getUpdateDirect(rh *ResourceHandler, key Key) ([]byte, error) { } return data, nil } - -type testResolver struct { - ens *ens.ENS -} - -func newTestResolver(ens *ens.ENS) *testResolver { - return &testResolver{ - ens: ens, - } -} - -func (r *testResolver) ValidateOwner(name string, address common.Address) (bool, error) { - addr, err := r.ens.Owner(ens.EnsNode(name)) - return addr == address, err -} diff --git a/swarm/storage/types.go b/swarm/storage/types.go index b655fe0e25..aa2973e0de 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/bmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/sha3" + "github.com/ethereum/go-ethereum/log" ) const MaxPO = 7 @@ -180,18 +181,18 @@ type Chunk struct { dbStoredC chan bool // never remove a chunk from memStore before it is written to dbStore dbStored bool dbStoredMu *sync.Mutex - errored bool // flag which is set when the chunk request has errored or timeouted + errored error // flag which is set when the chunk request has errored or timeouted erroredMu sync.Mutex } -func (c *Chunk) SetErrored(val bool) { +func (c *Chunk) SetErrored(err error) { c.erroredMu.Lock() defer c.erroredMu.Unlock() - c.errored = val + c.errored = err } -func (c *Chunk) GetErrored() bool { +func (c *Chunk) GetErrored() error { c.erroredMu.Lock() defer c.erroredMu.Unlock() @@ -257,6 +258,34 @@ func (self *LazyTestSectionReader) Size(chan bool) (int64, error) { return self.SectionReader.Size(), nil } +type StoreParams struct { + Hash SwarmHasher `toml:"-"` + DbCapacity uint64 + CacheCapacity uint + ChunkRequestsCacheCapacity uint + BaseKey []byte +} + +func NewDefaultStoreParams() *StoreParams { + return NewStoreParams(defaultLDBCapacity, defaultCacheCapacity, defaultChunkRequestsCacheCapacity, nil, nil) +} + +func NewStoreParams(ldbCap uint64, cacheCap uint, requestsCap uint, hash SwarmHasher, basekey []byte) *StoreParams { + if basekey == nil { + basekey = make([]byte, 32) + } + if hash == nil { + hash = MakeHashFunc("SHA3") + } + return &StoreParams{ + Hash: hash, + DbCapacity: ldbCap, + CacheCapacity: cacheCap, + ChunkRequestsCacheCapacity: requestsCap, + BaseKey: basekey, + } +} + type ChunkData []byte type Reference []byte @@ -285,3 +314,34 @@ func (c ChunkData) Size() int64 { func (c ChunkData) Data() []byte { return c[8:] } + +type ChunkValidator interface { + Validate(key Key, data []byte) bool +} + +// Provides method for validation of content address in chunks +// Holds the corresponding hasher to create the address +type ContentAddressValidator struct { + Hasher SwarmHasher +} + +// Constructor +func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator { + return &ContentAddressValidator{ + Hasher: hasher, + } +} + +// Validate that the given key is a valid content address for the given data +func (self *ContentAddressValidator) Validate(key Key, data []byte) bool { + hasher := self.Hasher() + hasher.ResetWithLength(data[:8]) + hasher.Write(data[8:]) + hash := hasher.Sum(nil) + + if !bytes.Equal(hash, key[:]) { + log.Error("invalid content address", "expected", fmt.Sprintf("%x", hash), "have", key) + return false + } + return true +} diff --git a/swarm/swarm.go b/swarm/swarm.go index d387780321..a1c96127f5 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -117,15 +117,6 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. } log.Debug(fmt.Sprintf("Setting up Swarm service components")) - hash := storage.MakeHashFunc(config.DPAParams.Hash) - self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, common.FromHex(config.BzzKey), mockStore) - if err != nil { - return - } - - // setup local store - log.Debug(fmt.Sprintf("Set up local storage")) - kp := network.NewKadParams() to := network.NewKademlia( common.FromHex(config.BzzKey), @@ -146,40 +137,16 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. HiveParams: config.HiveParams, } - db := storage.NewDBAPI(self.lstore) - delivery := stream.NewDelivery(to, db) // TODO: decide on intervals store file location stateStore, err := state.NewDBStore(filepath.Join(config.Path, "state-store.db")) if err != nil { return } - self.streamer = stream.NewRegistry(addr, delivery, db, stateStore, &stream.RegistryOptions{ - DoSync: true, - DoRetrieve: true, - SyncUpdateDelay: config.SyncUpdateDelay, - }) - - self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run) - - // set up DPA, the cloud storage local access layer - dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve) - log.Debug(fmt.Sprintf("-> Local Access to Swarm")) - // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage - self.dpa = storage.NewDPA(dpaChunkStore, self.config.DPAParams) - log.Debug(fmt.Sprintf("-> Content Store API")) - - // Pss = postal service over swarm (devp2p over bzz) - if self.config.PssEnabled { - pssparams := pss.NewPssParams(self.privateKey) - self.ps = pss.NewPss(to, pssparams) - if pss.IsActiveHandshake { - pss.SetHandshakeController(self.ps, pss.NewHandshakeParams()) - } - } // set up high level api //transactOpts := bind.NewKeyedTransactor(self.privateKey) var resolver *api.MultiResolver + var ensresolver *ens.ENS if len(config.EnsAPIs) > 0 { opts := []api.MultiResolverOption{} for _, c := range config.EnsAPIs { @@ -188,6 +155,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. if err != nil { return nil, err } + ensresolver = r.ENS opts = append(opts, api.MultiResolverOptionWithResolver(r, tld)) } @@ -195,19 +163,67 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. self.dns = resolver } + self.lstore, err = storage.NewLocalStore(config.LocalStoreParams, mockStore) + if err != nil { + return + } + var resourceHandler *storage.ResourceHandler // if use resource updates + if self.config.ResourceEnabled && resolver != nil { - resourceparams := &storage.ResourceHandlerParams{ - Validator: storage.NewENSValidator(config.EnsRoot, resolver, storage.NewGenericResourceSigner(self.privateKey)), + resolver.SetNameHash(ens.EnsNode) + rhparams := &storage.ResourceHandlerParams{ + // TODO: config parameter to set limits + QueryMaxPeriods: &storage.ResourceLookupParams{ + Limit: false, + }, + Signer: &storage.GenericResourceSigner{ + PrivKey: self.privateKey, + }, + EthClient: resolver, + EnsClient: ensresolver, } - resolver.SetNameHash(resourceparams.Validator.NameHash) - hashfunc := storage.MakeHashFunc(storage.SHA3Hash) - chunkStore := storage.NewResourceChunkStore(self.lstore, func(*storage.Chunk) error { return nil }) - resourceHandler, err = storage.NewResourceHandler(hashfunc, chunkStore, resolver, resourceparams) + resourceHandler, err = storage.NewResourceHandler(rhparams) if err != nil { return nil, err } + resourceHandler.SetStore(self.lstore) + } + + var validators []storage.ChunkValidator + validators = append(validators, storage.NewContentAddressValidator(storage.MakeHashFunc(storage.SHA3Hash))) + if resourceHandler != nil { + validators = append(validators, resourceHandler) + } + self.lstore.Validators = validators + + // setup local store + log.Debug(fmt.Sprintf("Set up local storage")) + + db := storage.NewDBAPI(self.lstore) + delivery := stream.NewDelivery(to, db) + + self.streamer = stream.NewRegistry(addr, delivery, db, stateStore, &stream.RegistryOptions{ + DoSync: true, + DoRetrieve: true, + SyncUpdateDelay: config.SyncUpdateDelay, + }) + + // set up DPA, the cloud storage local access layer + dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve) + log.Debug(fmt.Sprintf("-> Local Access to Swarm")) + // Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage + self.dpa = storage.NewDPA(dpaChunkStore, self.config.DPAParams) + log.Debug(fmt.Sprintf("-> Content Store API")) + + self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run) + + // Pss = postal service over swarm (devp2p over bzz) + pssparams := pss.NewPssParams(self.privateKey) + self.ps = pss.NewPss(to, pssparams) + if pss.IsActiveHandshake { + pss.SetHandshakeController(self.ps, pss.NewHandshakeParams()) } self.api = api.NewApi(self.dpa, self.dns, resourceHandler) diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index f253d13a66..a7baa6583b 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -47,12 +47,11 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { if err != nil { t.Fatal(err) } - storeparams := &storage.StoreParams{ - ChunkDbPath: dir, - DbCapacity: 5000000, - CacheCapacity: 5000, - } - localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, make([]byte, 32), nil) + storeparams := storage.NewDefaultLocalStoreParams() + storeparams.DbCapacity = 5000000 + storeparams.CacheCapacity = 5000 + storeparams.Init(dir) + localStore, err := storage.NewLocalStore(storeparams, nil) if err != nil { os.RemoveAll(dir) t.Fatal(err) @@ -64,8 +63,11 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { if err != nil { t.Fatal(err) } - - rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil, &storage.ResourceLookupParams{Limit: false}) + rhparams := &storage.ResourceHandlerParams{ + QueryMaxPeriods: &storage.ResourceLookupParams{}, + EthClient: &fakeBackend{}, + } + rh, err := storage.NewTestResourceHandler(resourceDir, rhparams) if err != nil { t.Fatal(err) } diff --git a/vendor/github.com/naoina/toml/encode.go b/vendor/github.com/naoina/toml/encode.go index ae6bfd575f..15602f005a 100644 --- a/vendor/github.com/naoina/toml/encode.go +++ b/vendor/github.com/naoina/toml/encode.go @@ -61,20 +61,26 @@ func (cfg *Config) NewEncoder(w io.Writer) *Encoder { // Encode writes the TOML of v to the stream. // See the documentation for Marshal for details about the conversion of Go values to TOML. func (e *Encoder) Encode(v interface{}) error { - rv := reflect.ValueOf(v) + var ( + buf = &tableBuf{typ: ast.TableTypeNormal} + rv = reflect.ValueOf(v) + err error + ) + for rv.Kind() == reflect.Ptr { if rv.IsNil() { return &marshalNilError{rv.Type()} } rv = rv.Elem() } - buf := &tableBuf{typ: ast.TableTypeNormal} - var err error + switch rv.Kind() { case reflect.Struct: err = buf.structFields(e.cfg, rv) case reflect.Map: err = buf.mapFields(e.cfg, rv) + case reflect.Interface: + return e.Encode(rv.Interface()) default: err = &marshalTableError{rv.Type()} } diff --git a/vendor/github.com/naoina/toml/parse.go b/vendor/github.com/naoina/toml/parse.go index e6f95001e5..de9108566b 100644 --- a/vendor/github.com/naoina/toml/parse.go +++ b/vendor/github.com/naoina/toml/parse.go @@ -97,6 +97,7 @@ type toml struct { currentTable *ast.Table s string key string + tableKeys []string val ast.Value arr *array stack []*stack @@ -180,12 +181,12 @@ func (p *tomlParser) SetArray(begin, end int) { } func (p *toml) SetTable(buf []rune, begin, end int) { - p.setTable(p.table, buf, begin, end) + rawName := string(buf[begin:end]) + p.setTable(p.table, rawName, p.tableKeys) + p.tableKeys = nil } -func (p *toml) setTable(parent *ast.Table, buf []rune, begin, end int) { - name := string(buf[begin:end]) - names := splitTableKey(name) +func (p *toml) setTable(parent *ast.Table, name string, names []string) { parent, err := p.lookupTable(parent, names[:len(names)-1]) if err != nil { p.Error(err) @@ -230,12 +231,12 @@ func (p *tomlParser) SetTableString(begin, end int) { } func (p *toml) SetArrayTable(buf []rune, begin, end int) { - p.setArrayTable(p.table, buf, begin, end) + rawName := string(buf[begin:end]) + p.setArrayTable(p.table, rawName, p.tableKeys) + p.tableKeys = nil } -func (p *toml) setArrayTable(parent *ast.Table, buf []rune, begin, end int) { - name := string(buf[begin:end]) - names := splitTableKey(name) +func (p *toml) setArrayTable(parent *ast.Table, name string, names []string) { parent, err := p.lookupTable(parent, names[:len(names)-1]) if err != nil { p.Error(err) @@ -260,11 +261,11 @@ func (p *toml) setArrayTable(parent *ast.Table, buf []rune, begin, end int) { func (p *toml) StartInlineTable() { p.skip = false p.stack = append(p.stack, &stack{p.key, p.currentTable}) - buf := []rune(p.key) + names := []string{p.key} if p.arr == nil { - p.setTable(p.currentTable, buf, 0, len(buf)) + p.setTable(p.currentTable, names[0], names) } else { - p.setArrayTable(p.currentTable, buf, 0, len(buf)) + p.setArrayTable(p.currentTable, names[0], names) } } @@ -282,6 +283,13 @@ func (p *toml) AddLineCount(i int) { func (p *toml) SetKey(buf []rune, begin, end int) { p.key = string(buf[begin:end]) + if len(p.key) > 0 && p.key[0] == '"' { + p.key = p.unquote(p.key) + } +} + +func (p *toml) AddTableKey() { + p.tableKeys = append(p.tableKeys, p.key) } func (p *toml) AddKeyValue() { @@ -352,25 +360,3 @@ func (p *toml) lookupTable(t *ast.Table, keys []string) (*ast.Table, error) { } return t, nil } - -func splitTableKey(tk string) []string { - key := make([]byte, 0, 1) - keys := make([]string, 0, 1) - inQuote := false - for i := 0; i < len(tk); i++ { - k := tk[i] - switch { - case k == tableSeparator && !inQuote: - keys = append(keys, string(key)) - key = key[:0] // reuse buffer. - case k == '"': - inQuote = !inQuote - case (k == ' ' || k == '\t') && !inQuote: - // skip. - default: - key = append(key, k) - } - } - keys = append(keys, string(key)) - return keys -} diff --git a/vendor/github.com/naoina/toml/parse.peg b/vendor/github.com/naoina/toml/parse.peg index da31dae309..860ada3732 100644 --- a/vendor/github.com/naoina/toml/parse.peg +++ b/vendor/github.com/naoina/toml/parse.peg @@ -29,7 +29,7 @@ key <- bareKey / quotedKey bareKey <- <[0-9A-Za-z\-_]+> { p.SetKey(p.buffer, begin, end) } -quotedKey <- '"' '"' { p.SetKey(p.buffer, begin-1, end+1) } +quotedKey <- < '"' basicChar* '"' > { p.SetKey(p.buffer, begin, end) } val <- ( { p.SetTime(begin, end) } @@ -55,7 +55,9 @@ inlineTable <- ( inlineTableKeyValues <- (keyval inlineTableValSep?)* -tableKey <- key (tableKeySep key)* +tableKey <- tableKeyComp (tableKeySep tableKeyComp)* + +tableKeyComp <- key { p.AddTableKey() } tableKeySep <- ws '.' ws @@ -117,7 +119,7 @@ timeNumoffset <- [\-+] timeHour ':' timeMinute timeOffset <- 'Z' / timeNumoffset partialTime <- timeHour ':' timeMinute ':' timeSecond timeSecfrac? fullDate <- dateFullYear '-' dateMonth '-' dateMDay -fullTime <- partialTime timeOffset +fullTime <- partialTime timeOffset? datetime <- (fullDate ('T' fullTime)?) / partialTime digit <- [0-9] diff --git a/vendor/github.com/naoina/toml/parse.peg.go b/vendor/github.com/naoina/toml/parse.peg.go index d7de73b19c..fa377b19bd 100644 --- a/vendor/github.com/naoina/toml/parse.peg.go +++ b/vendor/github.com/naoina/toml/parse.peg.go @@ -31,6 +31,7 @@ const ( ruleinlineTable ruleinlineTableKeyValues ruletableKey + ruletableKeyComp ruletableKeySep ruleinlineTableValSep ruleinteger @@ -99,6 +100,7 @@ const ( ruleAction22 ruleAction23 ruleAction24 + ruleAction25 ) var rul3s = [...]string{ @@ -120,6 +122,7 @@ var rul3s = [...]string{ "inlineTable", "inlineTableKeyValues", "tableKey", + "tableKeyComp", "tableKeySep", "inlineTableValSep", "integer", @@ -188,6 +191,7 @@ var rul3s = [...]string{ "Action22", "Action23", "Action24", + "Action25", } type token32 struct { @@ -304,7 +308,7 @@ type tomlParser struct { Buffer string buffer []rune - rules [86]func() bool + rules [88]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -409,7 +413,7 @@ func (p *tomlParser) Execute() { case ruleAction5: p.SetKey(p.buffer, begin, end) case ruleAction6: - p.SetKey(p.buffer, begin-1, end+1) + p.SetKey(p.buffer, begin, end) case ruleAction7: p.SetTime(begin, end) case ruleAction8: @@ -431,21 +435,23 @@ func (p *tomlParser) Execute() { case ruleAction16: p.EndInlineTable() case ruleAction17: - p.SetBasicString(p.buffer, begin, end) + p.AddTableKey() case ruleAction18: - p.SetMultilineString() + p.SetBasicString(p.buffer, begin, end) case ruleAction19: - p.AddMultilineBasicBody(p.buffer, begin, end) + p.SetMultilineString() case ruleAction20: - p.SetLiteralString(p.buffer, begin, end) + p.AddMultilineBasicBody(p.buffer, begin, end) case ruleAction21: - p.SetMultilineLiteralString(p.buffer, begin, end) + p.SetLiteralString(p.buffer, begin, end) case ruleAction22: - p.StartArray() + p.SetMultilineLiteralString(p.buffer, begin, end) case ruleAction23: - p.AddArrayVal() + p.StartArray() case ruleAction24: p.AddArrayVal() + case ruleAction25: + p.AddArrayVal() } } @@ -1069,15 +1075,12 @@ func (p *tomlParser) Init() { position, tokenIndex = position72, tokenIndex72 { position81 := position - if buffer[position] != rune('"') { - goto l70 - } - position++ { position82 := position - if !_rules[rulebasicChar]() { + if buffer[position] != rune('"') { goto l70 } + position++ l83: { position84, tokenIndex84 := position, tokenIndex @@ -1088,12 +1091,12 @@ func (p *tomlParser) Init() { l84: position, tokenIndex = position84, tokenIndex84 } + if buffer[position] != rune('"') { + goto l70 + } + position++ add(rulePegText, position82) } - if buffer[position] != rune('"') { - goto l70 - } - position++ { add(ruleAction6, position) } @@ -1110,7 +1113,7 @@ func (p *tomlParser) Init() { }, /* 8 bareKey <- <(<((&('_') '_') | (&('-') '-') | (&('a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z') [a-z]) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z') [A-Z]))+> Action5)> */ nil, - /* 9 quotedKey <- <('"' '"' Action6)> */ + /* 9 quotedKey <- <(<('"' basicChar* '"')> Action6)> */ nil, /* 10 val <- <(( Action7) / ( Action8) / ((&('{') inlineTable) | (&('[') ( Action12)) | (&('f' | 't') ( Action11)) | (&('"' | '\'') ( Action10)) | (&('+' | '-' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') ( Action9))))> */ func() bool { @@ -1177,49 +1180,56 @@ func (p *tomlParser) Init() { goto l101 } { - position104 := position + position104, tokenIndex104 := position, tokenIndex { - position105, tokenIndex105 := position, tokenIndex - if buffer[position] != rune('Z') { - goto l106 - } - position++ - goto l105 - l106: - position, tokenIndex = position105, tokenIndex105 + position106 := position { - position107 := position - { - position108, tokenIndex108 := position, tokenIndex - if buffer[position] != rune('-') { - goto l109 - } - position++ + position107, tokenIndex107 := position, tokenIndex + if buffer[position] != rune('Z') { goto l108 - l109: - position, tokenIndex = position108, tokenIndex108 - if buffer[position] != rune('+') { - goto l101 - } - position++ - } - l108: - if !_rules[ruletimeHour]() { - goto l101 - } - if buffer[position] != rune(':') { - goto l101 } position++ - if !_rules[ruletimeMinute]() { - goto l101 + goto l107 + l108: + position, tokenIndex = position107, tokenIndex107 + { + position109 := position + { + position110, tokenIndex110 := position, tokenIndex + if buffer[position] != rune('-') { + goto l111 + } + position++ + goto l110 + l111: + position, tokenIndex = position110, tokenIndex110 + if buffer[position] != rune('+') { + goto l104 + } + position++ + } + l110: + if !_rules[ruletimeHour]() { + goto l104 + } + if buffer[position] != rune(':') { + goto l104 + } + position++ + if !_rules[ruletimeMinute]() { + goto l104 + } + add(ruletimeNumoffset, position109) } - add(ruletimeNumoffset, position107) } + l107: + add(ruletimeOffset, position106) } - l105: - add(ruletimeOffset, position104) + goto l105 + l104: + position, tokenIndex = position104, tokenIndex104 } + l105: add(rulefullTime, position103) } goto l102 @@ -1246,33 +1256,20 @@ func (p *tomlParser) Init() { l91: position, tokenIndex = position90, tokenIndex90 { - position112 := position + position114 := position { - position113 := position + position115 := position if !_rules[ruleinteger]() { - goto l111 + goto l113 } { - position114, tokenIndex114 := position, tokenIndex + position116, tokenIndex116 := position, tokenIndex if !_rules[rulefrac]() { - goto l115 - } - { - position116, tokenIndex116 := position, tokenIndex - if !_rules[ruleexp]() { - goto l116 - } goto l117 - l116: - position, tokenIndex = position116, tokenIndex116 } - l117: - goto l114 - l115: - position, tokenIndex = position114, tokenIndex114 { position118, tokenIndex118 := position, tokenIndex - if !_rules[rulefrac]() { + if !_rules[ruleexp]() { goto l118 } goto l119 @@ -1280,26 +1277,39 @@ func (p *tomlParser) Init() { position, tokenIndex = position118, tokenIndex118 } l119: + goto l116 + l117: + position, tokenIndex = position116, tokenIndex116 + { + position120, tokenIndex120 := position, tokenIndex + if !_rules[rulefrac]() { + goto l120 + } + goto l121 + l120: + position, tokenIndex = position120, tokenIndex120 + } + l121: if !_rules[ruleexp]() { - goto l111 + goto l113 } } - l114: - add(rulefloat, position113) + l116: + add(rulefloat, position115) } - add(rulePegText, position112) + add(rulePegText, position114) } { add(ruleAction8, position) } goto l90 - l111: + l113: position, tokenIndex = position90, tokenIndex90 { switch buffer[position] { case '{': { - position122 := position + position124 := position if buffer[position] != rune('{') { goto l88 } @@ -1311,39 +1321,39 @@ func (p *tomlParser) Init() { goto l88 } { - position124 := position - l125: + position126 := position + l127: { - position126, tokenIndex126 := position, tokenIndex + position128, tokenIndex128 := position, tokenIndex if !_rules[rulekeyval]() { - goto l126 + goto l128 } { - position127, tokenIndex127 := position, tokenIndex + position129, tokenIndex129 := position, tokenIndex { - position129 := position + position131 := position if !_rules[rulews]() { - goto l127 + goto l129 } if buffer[position] != rune(',') { - goto l127 + goto l129 } position++ if !_rules[rulews]() { - goto l127 + goto l129 } - add(ruleinlineTableValSep, position129) + add(ruleinlineTableValSep, position131) } - goto l128 - l127: - position, tokenIndex = position127, tokenIndex127 + goto l130 + l129: + position, tokenIndex = position129, tokenIndex129 } + l130: + goto l127 l128: - goto l125 - l126: - position, tokenIndex = position126, tokenIndex126 + position, tokenIndex = position128, tokenIndex128 } - add(ruleinlineTableKeyValues, position124) + add(ruleinlineTableKeyValues, position126) } if !_rules[rulews]() { goto l88 @@ -1355,58 +1365,39 @@ func (p *tomlParser) Init() { { add(ruleAction16, position) } - add(ruleinlineTable, position122) + add(ruleinlineTable, position124) } break case '[': { - position131 := position + position133 := position { - position132 := position + position134 := position if buffer[position] != rune('[') { goto l88 } position++ { - add(ruleAction22, position) + add(ruleAction23, position) } if !_rules[rulewsnl]() { goto l88 } { - position134, tokenIndex134 := position, tokenIndex + position136, tokenIndex136 := position, tokenIndex { - position136 := position + position138 := position if !_rules[ruleval]() { - goto l134 + goto l136 } { - add(ruleAction23, position) + add(ruleAction24, position) } - l138: + l140: { - position139, tokenIndex139 := position, tokenIndex + position141, tokenIndex141 := position, tokenIndex if !_rules[rulewsnl]() { - goto l139 - } - { - position140, tokenIndex140 := position, tokenIndex - if !_rules[rulecomment]() { - goto l140 - } goto l141 - l140: - position, tokenIndex = position140, tokenIndex140 - } - l141: - if !_rules[rulewsnl]() { - goto l139 - } - if !_rules[rulearraySep]() { - goto l139 - } - if !_rules[rulewsnl]() { - goto l139 } { position142, tokenIndex142 := position, tokenIndex @@ -1419,37 +1410,43 @@ func (p *tomlParser) Init() { } l143: if !_rules[rulewsnl]() { - goto l139 + goto l141 } - if !_rules[ruleval]() { - goto l139 + if !_rules[rulearraySep]() { + goto l141 + } + if !_rules[rulewsnl]() { + goto l141 } { - add(ruleAction24, position) - } - goto l138 - l139: - position, tokenIndex = position139, tokenIndex139 - } - if !_rules[rulewsnl]() { - goto l134 - } - { - position145, tokenIndex145 := position, tokenIndex - if !_rules[rulearraySep]() { + position144, tokenIndex144 := position, tokenIndex + if !_rules[rulecomment]() { + goto l144 + } goto l145 + l144: + position, tokenIndex = position144, tokenIndex144 } - goto l146 l145: - position, tokenIndex = position145, tokenIndex145 + if !_rules[rulewsnl]() { + goto l141 + } + if !_rules[ruleval]() { + goto l141 + } + { + add(ruleAction25, position) + } + goto l140 + l141: + position, tokenIndex = position141, tokenIndex141 } - l146: if !_rules[rulewsnl]() { - goto l134 + goto l136 } { position147, tokenIndex147 := position, tokenIndex - if !_rules[rulecomment]() { + if !_rules[rulearraySep]() { goto l147 } goto l148 @@ -1457,13 +1454,26 @@ func (p *tomlParser) Init() { position, tokenIndex = position147, tokenIndex147 } l148: - add(rulearrayValues, position136) + if !_rules[rulewsnl]() { + goto l136 + } + { + position149, tokenIndex149 := position, tokenIndex + if !_rules[rulecomment]() { + goto l149 + } + goto l150 + l149: + position, tokenIndex = position149, tokenIndex149 + } + l150: + add(rulearrayValues, position138) } - goto l135 - l134: - position, tokenIndex = position134, tokenIndex134 + goto l137 + l136: + position, tokenIndex = position136, tokenIndex136 } - l135: + l137: if !_rules[rulewsnl]() { goto l88 } @@ -1471,9 +1481,9 @@ func (p *tomlParser) Init() { goto l88 } position++ - add(rulearray, position132) + add(rulearray, position134) } - add(rulePegText, position131) + add(rulePegText, position133) } { add(ruleAction12, position) @@ -1481,30 +1491,30 @@ func (p *tomlParser) Init() { break case 'f', 't': { - position150 := position + position152 := position { - position151 := position + position153 := position { - position152, tokenIndex152 := position, tokenIndex + position154, tokenIndex154 := position, tokenIndex if buffer[position] != rune('t') { - goto l153 + goto l155 } position++ if buffer[position] != rune('r') { - goto l153 + goto l155 } position++ if buffer[position] != rune('u') { - goto l153 + goto l155 } position++ if buffer[position] != rune('e') { - goto l153 + goto l155 } position++ - goto l152 - l153: - position, tokenIndex = position152, tokenIndex152 + goto l154 + l155: + position, tokenIndex = position154, tokenIndex154 if buffer[position] != rune('f') { goto l88 } @@ -1526,10 +1536,10 @@ func (p *tomlParser) Init() { } position++ } - l152: - add(ruleboolean, position151) + l154: + add(ruleboolean, position153) } - add(rulePegText, position150) + add(rulePegText, position152) } { add(ruleAction11, position) @@ -1537,278 +1547,278 @@ func (p *tomlParser) Init() { break case '"', '\'': { - position155 := position + position157 := position { - position156 := position + position158 := position { - position157, tokenIndex157 := position, tokenIndex + position159, tokenIndex159 := position, tokenIndex { - position159 := position + position161 := position if buffer[position] != rune('\'') { - goto l158 + goto l160 } position++ if buffer[position] != rune('\'') { - goto l158 + goto l160 } position++ if buffer[position] != rune('\'') { - goto l158 + goto l160 } position++ { - position160 := position + position162 := position { - position161 := position - l162: + position163 := position + l164: { - position163, tokenIndex163 := position, tokenIndex + position165, tokenIndex165 := position, tokenIndex { - position164, tokenIndex164 := position, tokenIndex + position166, tokenIndex166 := position, tokenIndex if buffer[position] != rune('\'') { - goto l164 + goto l166 } position++ if buffer[position] != rune('\'') { - goto l164 + goto l166 } position++ if buffer[position] != rune('\'') { - goto l164 + goto l166 } position++ - goto l163 - l164: - position, tokenIndex = position164, tokenIndex164 + goto l165 + l166: + position, tokenIndex = position166, tokenIndex166 } { - position165, tokenIndex165 := position, tokenIndex + position167, tokenIndex167 := position, tokenIndex { - position167 := position + position169 := position { - position168, tokenIndex168 := position, tokenIndex + position170, tokenIndex170 := position, tokenIndex if buffer[position] != rune('\t') { - goto l169 + goto l171 } position++ - goto l168 - l169: - position, tokenIndex = position168, tokenIndex168 + goto l170 + l171: + position, tokenIndex = position170, tokenIndex170 if c := buffer[position]; c < rune(' ') || c > rune('\U0010ffff') { - goto l166 + goto l168 } position++ } - l168: - add(rulemlLiteralChar, position167) + l170: + add(rulemlLiteralChar, position169) } - goto l165 - l166: - position, tokenIndex = position165, tokenIndex165 + goto l167 + l168: + position, tokenIndex = position167, tokenIndex167 if !_rules[rulenewline]() { - goto l163 + goto l165 } } + l167: + goto l164 l165: - goto l162 - l163: - position, tokenIndex = position163, tokenIndex163 + position, tokenIndex = position165, tokenIndex165 } - add(rulemlLiteralBody, position161) + add(rulemlLiteralBody, position163) } - add(rulePegText, position160) + add(rulePegText, position162) } if buffer[position] != rune('\'') { - goto l158 + goto l160 } position++ if buffer[position] != rune('\'') { - goto l158 + goto l160 } position++ if buffer[position] != rune('\'') { - goto l158 + goto l160 } position++ { - add(ruleAction21, position) + add(ruleAction22, position) } - add(rulemlLiteralString, position159) + add(rulemlLiteralString, position161) } - goto l157 - l158: - position, tokenIndex = position157, tokenIndex157 + goto l159 + l160: + position, tokenIndex = position159, tokenIndex159 { - position172 := position + position174 := position if buffer[position] != rune('\'') { - goto l171 + goto l173 } position++ { - position173 := position - l174: + position175 := position + l176: { - position175, tokenIndex175 := position, tokenIndex + position177, tokenIndex177 := position, tokenIndex { - position176 := position + position178 := position { switch buffer[position] { case '\t': if buffer[position] != rune('\t') { - goto l175 + goto l177 } position++ break case ' ', '!', '"', '#', '$', '%', '&': if c := buffer[position]; c < rune(' ') || c > rune('&') { - goto l175 + goto l177 } position++ break default: if c := buffer[position]; c < rune('(') || c > rune('\U0010ffff') { - goto l175 + goto l177 } position++ break } } - add(ruleliteralChar, position176) + add(ruleliteralChar, position178) } - goto l174 - l175: - position, tokenIndex = position175, tokenIndex175 + goto l176 + l177: + position, tokenIndex = position177, tokenIndex177 } - add(rulePegText, position173) + add(rulePegText, position175) } if buffer[position] != rune('\'') { - goto l171 + goto l173 } position++ { - add(ruleAction20, position) + add(ruleAction21, position) } - add(ruleliteralString, position172) + add(ruleliteralString, position174) } - goto l157 - l171: - position, tokenIndex = position157, tokenIndex157 + goto l159 + l173: + position, tokenIndex = position159, tokenIndex159 { - position180 := position + position182 := position if buffer[position] != rune('"') { - goto l179 + goto l181 } position++ if buffer[position] != rune('"') { - goto l179 + goto l181 } position++ if buffer[position] != rune('"') { - goto l179 + goto l181 } position++ { - position181 := position - l182: + position183 := position + l184: { - position183, tokenIndex183 := position, tokenIndex + position185, tokenIndex185 := position, tokenIndex { - position184, tokenIndex184 := position, tokenIndex + position186, tokenIndex186 := position, tokenIndex { - position186 := position + position188 := position { - position187, tokenIndex187 := position, tokenIndex + position189, tokenIndex189 := position, tokenIndex if !_rules[rulebasicChar]() { - goto l188 + goto l190 } - goto l187 - l188: - position, tokenIndex = position187, tokenIndex187 + goto l189 + l190: + position, tokenIndex = position189, tokenIndex189 if !_rules[rulenewline]() { - goto l185 + goto l187 } } - l187: - add(rulePegText, position186) + l189: + add(rulePegText, position188) } { - add(ruleAction19, position) + add(ruleAction20, position) } - goto l184 - l185: - position, tokenIndex = position184, tokenIndex184 + goto l186 + l187: + position, tokenIndex = position186, tokenIndex186 if !_rules[ruleescape]() { - goto l183 + goto l185 } if !_rules[rulenewline]() { - goto l183 + goto l185 } if !_rules[rulewsnl]() { - goto l183 + goto l185 } } - l184: - goto l182 - l183: - position, tokenIndex = position183, tokenIndex183 + l186: + goto l184 + l185: + position, tokenIndex = position185, tokenIndex185 } - add(rulemlBasicBody, position181) + add(rulemlBasicBody, position183) } if buffer[position] != rune('"') { - goto l179 + goto l181 } position++ if buffer[position] != rune('"') { - goto l179 + goto l181 } position++ if buffer[position] != rune('"') { - goto l179 + goto l181 } position++ + { + add(ruleAction19, position) + } + add(rulemlBasicString, position182) + } + goto l159 + l181: + position, tokenIndex = position159, tokenIndex159 + { + position193 := position + { + position194 := position + if buffer[position] != rune('"') { + goto l88 + } + position++ + l195: + { + position196, tokenIndex196 := position, tokenIndex + if !_rules[rulebasicChar]() { + goto l196 + } + goto l195 + l196: + position, tokenIndex = position196, tokenIndex196 + } + if buffer[position] != rune('"') { + goto l88 + } + position++ + add(rulePegText, position194) + } { add(ruleAction18, position) } - add(rulemlBasicString, position180) - } - goto l157 - l179: - position, tokenIndex = position157, tokenIndex157 - { - position191 := position - { - position192 := position - if buffer[position] != rune('"') { - goto l88 - } - position++ - l193: - { - position194, tokenIndex194 := position, tokenIndex - if !_rules[rulebasicChar]() { - goto l194 - } - goto l193 - l194: - position, tokenIndex = position194, tokenIndex194 - } - if buffer[position] != rune('"') { - goto l88 - } - position++ - add(rulePegText, position192) - } - { - add(ruleAction17, position) - } - add(rulebasicString, position191) + add(rulebasicString, position193) } } - l157: - add(rulestring, position156) + l159: + add(rulestring, position158) } - add(rulePegText, position155) + add(rulePegText, position157) } { add(ruleAction10, position) @@ -1816,11 +1826,11 @@ func (p *tomlParser) Init() { break default: { - position197 := position + position199 := position if !_rules[ruleinteger]() { goto l88 } - add(rulePegText, position197) + add(rulePegText, position199) } { add(ruleAction9, position) @@ -1848,708 +1858,728 @@ func (p *tomlParser) Init() { nil, /* 15 inlineTableKeyValues <- <(keyval inlineTableValSep?)*> */ nil, - /* 16 tableKey <- <(key (tableKeySep key)*)> */ + /* 16 tableKey <- <(tableKeyComp (tableKeySep tableKeyComp)*)> */ func() bool { - position204, tokenIndex204 := position, tokenIndex + position206, tokenIndex206 := position, tokenIndex { - position205 := position - if !_rules[rulekey]() { - goto l204 + position207 := position + if !_rules[ruletableKeyComp]() { + goto l206 } - l206: + l208: { - position207, tokenIndex207 := position, tokenIndex + position209, tokenIndex209 := position, tokenIndex { - position208 := position + position210 := position if !_rules[rulews]() { - goto l207 + goto l209 } if buffer[position] != rune('.') { - goto l207 + goto l209 } position++ if !_rules[rulews]() { - goto l207 + goto l209 } - add(ruletableKeySep, position208) + add(ruletableKeySep, position210) } - if !_rules[rulekey]() { - goto l207 + if !_rules[ruletableKeyComp]() { + goto l209 } - goto l206 - l207: - position, tokenIndex = position207, tokenIndex207 + goto l208 + l209: + position, tokenIndex = position209, tokenIndex209 } - add(ruletableKey, position205) + add(ruletableKey, position207) } return true - l204: - position, tokenIndex = position204, tokenIndex204 + l206: + position, tokenIndex = position206, tokenIndex206 return false }, - /* 17 tableKeySep <- <(ws '.' ws)> */ - nil, - /* 18 inlineTableValSep <- <(ws ',' ws)> */ - nil, - /* 19 integer <- <(('-' / '+')? int)> */ + /* 17 tableKeyComp <- <(key Action17)> */ func() bool { position211, tokenIndex211 := position, tokenIndex { position212 := position - { - position213, tokenIndex213 := position, tokenIndex - { - position215, tokenIndex215 := position, tokenIndex - if buffer[position] != rune('-') { - goto l216 - } - position++ - goto l215 - l216: - position, tokenIndex = position215, tokenIndex215 - if buffer[position] != rune('+') { - goto l213 - } - position++ - } - l215: - goto l214 - l213: - position, tokenIndex = position213, tokenIndex213 + if !_rules[rulekey]() { + goto l211 } - l214: { - position217 := position - { - position218, tokenIndex218 := position, tokenIndex - if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l219 - } - position++ - { - position222, tokenIndex222 := position, tokenIndex - if !_rules[ruledigit]() { - goto l223 - } - goto l222 - l223: - position, tokenIndex = position222, tokenIndex222 - if buffer[position] != rune('_') { - goto l219 - } - position++ - if !_rules[ruledigit]() { - goto l219 - } - } - l222: - l220: - { - position221, tokenIndex221 := position, tokenIndex - { - position224, tokenIndex224 := position, tokenIndex - if !_rules[ruledigit]() { - goto l225 - } - goto l224 - l225: - position, tokenIndex = position224, tokenIndex224 - if buffer[position] != rune('_') { - goto l221 - } - position++ - if !_rules[ruledigit]() { - goto l221 - } - } - l224: - goto l220 - l221: - position, tokenIndex = position221, tokenIndex221 - } - goto l218 - l219: - position, tokenIndex = position218, tokenIndex218 - if !_rules[ruledigit]() { - goto l211 - } - } - l218: - add(ruleint, position217) + add(ruleAction17, position) } - add(ruleinteger, position212) + add(ruletableKeyComp, position212) } return true l211: position, tokenIndex = position211, tokenIndex211 return false }, - /* 20 int <- <(([1-9] (digit / ('_' digit))+) / digit)> */ + /* 18 tableKeySep <- <(ws '.' ws)> */ nil, - /* 21 float <- <(integer ((frac exp?) / (frac? exp)))> */ + /* 19 inlineTableValSep <- <(ws ',' ws)> */ nil, - /* 22 frac <- <('.' digit (digit / ('_' digit))*)> */ + /* 20 integer <- <(('-' / '+')? int)> */ func() bool { - position228, tokenIndex228 := position, tokenIndex + position216, tokenIndex216 := position, tokenIndex { - position229 := position + position217 := position + { + position218, tokenIndex218 := position, tokenIndex + { + position220, tokenIndex220 := position, tokenIndex + if buffer[position] != rune('-') { + goto l221 + } + position++ + goto l220 + l221: + position, tokenIndex = position220, tokenIndex220 + if buffer[position] != rune('+') { + goto l218 + } + position++ + } + l220: + goto l219 + l218: + position, tokenIndex = position218, tokenIndex218 + } + l219: + { + position222 := position + { + position223, tokenIndex223 := position, tokenIndex + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l224 + } + position++ + { + position227, tokenIndex227 := position, tokenIndex + if !_rules[ruledigit]() { + goto l228 + } + goto l227 + l228: + position, tokenIndex = position227, tokenIndex227 + if buffer[position] != rune('_') { + goto l224 + } + position++ + if !_rules[ruledigit]() { + goto l224 + } + } + l227: + l225: + { + position226, tokenIndex226 := position, tokenIndex + { + position229, tokenIndex229 := position, tokenIndex + if !_rules[ruledigit]() { + goto l230 + } + goto l229 + l230: + position, tokenIndex = position229, tokenIndex229 + if buffer[position] != rune('_') { + goto l226 + } + position++ + if !_rules[ruledigit]() { + goto l226 + } + } + l229: + goto l225 + l226: + position, tokenIndex = position226, tokenIndex226 + } + goto l223 + l224: + position, tokenIndex = position223, tokenIndex223 + if !_rules[ruledigit]() { + goto l216 + } + } + l223: + add(ruleint, position222) + } + add(ruleinteger, position217) + } + return true + l216: + position, tokenIndex = position216, tokenIndex216 + return false + }, + /* 21 int <- <(([1-9] (digit / ('_' digit))+) / digit)> */ + nil, + /* 22 float <- <(integer ((frac exp?) / (frac? exp)))> */ + nil, + /* 23 frac <- <('.' digit (digit / ('_' digit))*)> */ + func() bool { + position233, tokenIndex233 := position, tokenIndex + { + position234 := position if buffer[position] != rune('.') { - goto l228 + goto l233 } position++ if !_rules[ruledigit]() { - goto l228 + goto l233 } - l230: - { - position231, tokenIndex231 := position, tokenIndex - { - position232, tokenIndex232 := position, tokenIndex - if !_rules[ruledigit]() { - goto l233 - } - goto l232 - l233: - position, tokenIndex = position232, tokenIndex232 - if buffer[position] != rune('_') { - goto l231 - } - position++ - if !_rules[ruledigit]() { - goto l231 - } - } - l232: - goto l230 - l231: - position, tokenIndex = position231, tokenIndex231 - } - add(rulefrac, position229) - } - return true - l228: - position, tokenIndex = position228, tokenIndex228 - return false - }, - /* 23 exp <- <(('e' / 'E') ('-' / '+')? digit (digit / ('_' digit))*)> */ - func() bool { - position234, tokenIndex234 := position, tokenIndex - { - position235 := position + l235: { position236, tokenIndex236 := position, tokenIndex - if buffer[position] != rune('e') { - goto l237 - } - position++ - goto l236 - l237: - position, tokenIndex = position236, tokenIndex236 - if buffer[position] != rune('E') { - goto l234 - } - position++ - } - l236: - { - position238, tokenIndex238 := position, tokenIndex { - position240, tokenIndex240 := position, tokenIndex - if buffer[position] != rune('-') { - goto l241 - } - position++ - goto l240 - l241: - position, tokenIndex = position240, tokenIndex240 - if buffer[position] != rune('+') { + position237, tokenIndex237 := position, tokenIndex + if !_rules[ruledigit]() { goto l238 } + goto l237 + l238: + position, tokenIndex = position237, tokenIndex237 + if buffer[position] != rune('_') { + goto l236 + } position++ + if !_rules[ruledigit]() { + goto l236 + } } - l240: - goto l239 - l238: - position, tokenIndex = position238, tokenIndex238 + l237: + goto l235 + l236: + position, tokenIndex = position236, tokenIndex236 } - l239: - if !_rules[ruledigit]() { - goto l234 + add(rulefrac, position234) + } + return true + l233: + position, tokenIndex = position233, tokenIndex233 + return false + }, + /* 24 exp <- <(('e' / 'E') ('-' / '+')? digit (digit / ('_' digit))*)> */ + func() bool { + position239, tokenIndex239 := position, tokenIndex + { + position240 := position + { + position241, tokenIndex241 := position, tokenIndex + if buffer[position] != rune('e') { + goto l242 + } + position++ + goto l241 + l242: + position, tokenIndex = position241, tokenIndex241 + if buffer[position] != rune('E') { + goto l239 + } + position++ } - l242: + l241: { position243, tokenIndex243 := position, tokenIndex { - position244, tokenIndex244 := position, tokenIndex - if !_rules[ruledigit]() { - goto l245 + position245, tokenIndex245 := position, tokenIndex + if buffer[position] != rune('-') { + goto l246 } - goto l244 - l245: - position, tokenIndex = position244, tokenIndex244 - if buffer[position] != rune('_') { + position++ + goto l245 + l246: + position, tokenIndex = position245, tokenIndex245 + if buffer[position] != rune('+') { goto l243 } position++ - if !_rules[ruledigit]() { - goto l243 - } } - l244: - goto l242 + l245: + goto l244 l243: position, tokenIndex = position243, tokenIndex243 } - add(ruleexp, position235) + l244: + if !_rules[ruledigit]() { + goto l239 + } + l247: + { + position248, tokenIndex248 := position, tokenIndex + { + position249, tokenIndex249 := position, tokenIndex + if !_rules[ruledigit]() { + goto l250 + } + goto l249 + l250: + position, tokenIndex = position249, tokenIndex249 + if buffer[position] != rune('_') { + goto l248 + } + position++ + if !_rules[ruledigit]() { + goto l248 + } + } + l249: + goto l247 + l248: + position, tokenIndex = position248, tokenIndex248 + } + add(ruleexp, position240) } return true - l234: - position, tokenIndex = position234, tokenIndex234 + l239: + position, tokenIndex = position239, tokenIndex239 return false }, - /* 24 string <- <(mlLiteralString / literalString / mlBasicString / basicString)> */ + /* 25 string <- <(mlLiteralString / literalString / mlBasicString / basicString)> */ nil, - /* 25 basicString <- <(<('"' basicChar* '"')> Action17)> */ + /* 26 basicString <- <(<('"' basicChar* '"')> Action18)> */ nil, - /* 26 basicChar <- <(basicUnescaped / escaped)> */ + /* 27 basicChar <- <(basicUnescaped / escaped)> */ func() bool { - position248, tokenIndex248 := position, tokenIndex + position253, tokenIndex253 := position, tokenIndex { - position249 := position + position254 := position { - position250, tokenIndex250 := position, tokenIndex + position255, tokenIndex255 := position, tokenIndex { - position252 := position + position257 := position { switch buffer[position] { case ' ', '!': if c := buffer[position]; c < rune(' ') || c > rune('!') { - goto l251 + goto l256 } position++ break case '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[': if c := buffer[position]; c < rune('#') || c > rune('[') { - goto l251 + goto l256 } position++ break default: if c := buffer[position]; c < rune(']') || c > rune('\U0010ffff') { - goto l251 + goto l256 } position++ break } } - add(rulebasicUnescaped, position252) + add(rulebasicUnescaped, position257) } - goto l250 - l251: - position, tokenIndex = position250, tokenIndex250 + goto l255 + l256: + position, tokenIndex = position255, tokenIndex255 { - position254 := position + position259 := position if !_rules[ruleescape]() { - goto l248 + goto l253 } { switch buffer[position] { case 'U': if buffer[position] != rune('U') { - goto l248 + goto l253 } position++ if !_rules[rulehexQuad]() { - goto l248 + goto l253 } if !_rules[rulehexQuad]() { - goto l248 + goto l253 } break case 'u': if buffer[position] != rune('u') { - goto l248 + goto l253 } position++ if !_rules[rulehexQuad]() { - goto l248 + goto l253 } break case '\\': if buffer[position] != rune('\\') { - goto l248 + goto l253 } position++ break case '/': if buffer[position] != rune('/') { - goto l248 + goto l253 } position++ break case '"': if buffer[position] != rune('"') { - goto l248 + goto l253 } position++ break case 'r': if buffer[position] != rune('r') { - goto l248 + goto l253 } position++ break case 'f': if buffer[position] != rune('f') { - goto l248 + goto l253 } position++ break case 'n': if buffer[position] != rune('n') { - goto l248 + goto l253 } position++ break case 't': if buffer[position] != rune('t') { - goto l248 + goto l253 } position++ break default: if buffer[position] != rune('b') { - goto l248 + goto l253 } position++ break } } - add(ruleescaped, position254) + add(ruleescaped, position259) } } - l250: - add(rulebasicChar, position249) + l255: + add(rulebasicChar, position254) } return true - l248: - position, tokenIndex = position248, tokenIndex248 + l253: + position, tokenIndex = position253, tokenIndex253 return false }, - /* 27 escaped <- <(escape ((&('U') ('U' hexQuad hexQuad)) | (&('u') ('u' hexQuad)) | (&('\\') '\\') | (&('/') '/') | (&('"') '"') | (&('r') 'r') | (&('f') 'f') | (&('n') 'n') | (&('t') 't') | (&('b') 'b')))> */ + /* 28 escaped <- <(escape ((&('U') ('U' hexQuad hexQuad)) | (&('u') ('u' hexQuad)) | (&('\\') '\\') | (&('/') '/') | (&('"') '"') | (&('r') 'r') | (&('f') 'f') | (&('n') 'n') | (&('t') 't') | (&('b') 'b')))> */ nil, - /* 28 basicUnescaped <- <((&(' ' | '!') [ -!]) | (&('#' | '$' | '%' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '[') [#-[]) | (&(']' | '^' | '_' | '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | '{' | '|' | '}' | '~' | '\u007f' | '\u0080' | '\u0081' | '\u0082' | '\u0083' | '\u0084' | '\u0085' | '\u0086' | '\u0087' | '\u0088' | '\u0089' | '\u008a' | '\u008b' | '\u008c' | '\u008d' | '\u008e' | '\u008f' | '\u0090' | '\u0091' | '\u0092' | '\u0093' | '\u0094' | '\u0095' | '\u0096' | '\u0097' | '\u0098' | '\u0099' | '\u009a' | '\u009b' | '\u009c' | '\u009d' | '\u009e' | '\u009f' | '\u00a0' | '¡' | '¢' | '£' | '¤' | '¥' | '¦' | '§' | '¨' | '©' | 'ª' | '«' | '¬' | '\u00ad' | '®' | '¯' | '°' | '±' | '²' | '³' | '´' | 'µ' | '¶' | '·' | '¸' | '¹' | 'º' | '»' | '¼' | '½' | '¾' | '¿' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'Æ' | 'Ç' | 'È' | 'É' | 'Ê' | 'Ë' | 'Ì' | 'Í' | 'Î' | 'Ï' | 'Ð' | 'Ñ' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | '×' | 'Ø' | 'Ù' | 'Ú' | 'Û' | 'Ü' | 'Ý' | 'Þ' | 'ß' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'æ' | 'ç' | 'è' | 'é' | 'ê' | 'ë' | 'ì' | 'í' | 'î' | 'ï' | 'ð' | 'ñ' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | '÷' | 'ø' | 'ù' | 'ú' | 'û' | 'ü' | 'ý' | 'þ' | 'ÿ') []-\U0010ffff]))> */ + /* 29 basicUnescaped <- <((&(' ' | '!') [ -!]) | (&('#' | '$' | '%' | '&' | '\'' | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '[') [#-[]) | (&(']' | '^' | '_' | '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | '{' | '|' | '}' | '~' | '\u007f' | '\u0080' | '\u0081' | '\u0082' | '\u0083' | '\u0084' | '\u0085' | '\u0086' | '\u0087' | '\u0088' | '\u0089' | '\u008a' | '\u008b' | '\u008c' | '\u008d' | '\u008e' | '\u008f' | '\u0090' | '\u0091' | '\u0092' | '\u0093' | '\u0094' | '\u0095' | '\u0096' | '\u0097' | '\u0098' | '\u0099' | '\u009a' | '\u009b' | '\u009c' | '\u009d' | '\u009e' | '\u009f' | '\u00a0' | '¡' | '¢' | '£' | '¤' | '¥' | '¦' | '§' | '¨' | '©' | 'ª' | '«' | '¬' | '\u00ad' | '®' | '¯' | '°' | '±' | '²' | '³' | '´' | 'µ' | '¶' | '·' | '¸' | '¹' | 'º' | '»' | '¼' | '½' | '¾' | '¿' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'Æ' | 'Ç' | 'È' | 'É' | 'Ê' | 'Ë' | 'Ì' | 'Í' | 'Î' | 'Ï' | 'Ð' | 'Ñ' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | '×' | 'Ø' | 'Ù' | 'Ú' | 'Û' | 'Ü' | 'Ý' | 'Þ' | 'ß' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'æ' | 'ç' | 'è' | 'é' | 'ê' | 'ë' | 'ì' | 'í' | 'î' | 'ï' | 'ð' | 'ñ' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | '÷' | 'ø' | 'ù' | 'ú' | 'û' | 'ü' | 'ý' | 'þ' | 'ÿ') []-\U0010ffff]))> */ nil, - /* 29 escape <- <'\\'> */ + /* 30 escape <- <'\\'> */ func() bool { - position258, tokenIndex258 := position, tokenIndex + position263, tokenIndex263 := position, tokenIndex { - position259 := position + position264 := position if buffer[position] != rune('\\') { - goto l258 + goto l263 } position++ - add(ruleescape, position259) + add(ruleescape, position264) } return true - l258: - position, tokenIndex = position258, tokenIndex258 + l263: + position, tokenIndex = position263, tokenIndex263 return false }, - /* 30 mlBasicString <- <('"' '"' '"' mlBasicBody ('"' '"' '"') Action18)> */ + /* 31 mlBasicString <- <('"' '"' '"' mlBasicBody ('"' '"' '"') Action19)> */ nil, - /* 31 mlBasicBody <- <((<(basicChar / newline)> Action19) / (escape newline wsnl))*> */ + /* 32 mlBasicBody <- <((<(basicChar / newline)> Action20) / (escape newline wsnl))*> */ nil, - /* 32 literalString <- <('\'' '\'' Action20)> */ + /* 33 literalString <- <('\'' '\'' Action21)> */ nil, - /* 33 literalChar <- <((&('\t') '\t') | (&(' ' | '!' | '"' | '#' | '$' | '%' | '&') [ -&]) | (&('(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '[' | '\\' | ']' | '^' | '_' | '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | '{' | '|' | '}' | '~' | '\u007f' | '\u0080' | '\u0081' | '\u0082' | '\u0083' | '\u0084' | '\u0085' | '\u0086' | '\u0087' | '\u0088' | '\u0089' | '\u008a' | '\u008b' | '\u008c' | '\u008d' | '\u008e' | '\u008f' | '\u0090' | '\u0091' | '\u0092' | '\u0093' | '\u0094' | '\u0095' | '\u0096' | '\u0097' | '\u0098' | '\u0099' | '\u009a' | '\u009b' | '\u009c' | '\u009d' | '\u009e' | '\u009f' | '\u00a0' | '¡' | '¢' | '£' | '¤' | '¥' | '¦' | '§' | '¨' | '©' | 'ª' | '«' | '¬' | '\u00ad' | '®' | '¯' | '°' | '±' | '²' | '³' | '´' | 'µ' | '¶' | '·' | '¸' | '¹' | 'º' | '»' | '¼' | '½' | '¾' | '¿' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'Æ' | 'Ç' | 'È' | 'É' | 'Ê' | 'Ë' | 'Ì' | 'Í' | 'Î' | 'Ï' | 'Ð' | 'Ñ' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | '×' | 'Ø' | 'Ù' | 'Ú' | 'Û' | 'Ü' | 'Ý' | 'Þ' | 'ß' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'æ' | 'ç' | 'è' | 'é' | 'ê' | 'ë' | 'ì' | 'í' | 'î' | 'ï' | 'ð' | 'ñ' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | '÷' | 'ø' | 'ù' | 'ú' | 'û' | 'ü' | 'ý' | 'þ' | 'ÿ') [(-\U0010ffff]))> */ + /* 34 literalChar <- <((&('\t') '\t') | (&(' ' | '!' | '"' | '#' | '$' | '%' | '&') [ -&]) | (&('(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' | ':' | ';' | '<' | '=' | '>' | '?' | '@' | 'A' | 'B' | 'C' | 'D' | 'E' | 'F' | 'G' | 'H' | 'I' | 'J' | 'K' | 'L' | 'M' | 'N' | 'O' | 'P' | 'Q' | 'R' | 'S' | 'T' | 'U' | 'V' | 'W' | 'X' | 'Y' | 'Z' | '[' | '\\' | ']' | '^' | '_' | '`' | 'a' | 'b' | 'c' | 'd' | 'e' | 'f' | 'g' | 'h' | 'i' | 'j' | 'k' | 'l' | 'm' | 'n' | 'o' | 'p' | 'q' | 'r' | 's' | 't' | 'u' | 'v' | 'w' | 'x' | 'y' | 'z' | '{' | '|' | '}' | '~' | '\u007f' | '\u0080' | '\u0081' | '\u0082' | '\u0083' | '\u0084' | '\u0085' | '\u0086' | '\u0087' | '\u0088' | '\u0089' | '\u008a' | '\u008b' | '\u008c' | '\u008d' | '\u008e' | '\u008f' | '\u0090' | '\u0091' | '\u0092' | '\u0093' | '\u0094' | '\u0095' | '\u0096' | '\u0097' | '\u0098' | '\u0099' | '\u009a' | '\u009b' | '\u009c' | '\u009d' | '\u009e' | '\u009f' | '\u00a0' | '¡' | '¢' | '£' | '¤' | '¥' | '¦' | '§' | '¨' | '©' | 'ª' | '«' | '¬' | '\u00ad' | '®' | '¯' | '°' | '±' | '²' | '³' | '´' | 'µ' | '¶' | '·' | '¸' | '¹' | 'º' | '»' | '¼' | '½' | '¾' | '¿' | 'À' | 'Á' | 'Â' | 'Ã' | 'Ä' | 'Å' | 'Æ' | 'Ç' | 'È' | 'É' | 'Ê' | 'Ë' | 'Ì' | 'Í' | 'Î' | 'Ï' | 'Ð' | 'Ñ' | 'Ò' | 'Ó' | 'Ô' | 'Õ' | 'Ö' | '×' | 'Ø' | 'Ù' | 'Ú' | 'Û' | 'Ü' | 'Ý' | 'Þ' | 'ß' | 'à' | 'á' | 'â' | 'ã' | 'ä' | 'å' | 'æ' | 'ç' | 'è' | 'é' | 'ê' | 'ë' | 'ì' | 'í' | 'î' | 'ï' | 'ð' | 'ñ' | 'ò' | 'ó' | 'ô' | 'õ' | 'ö' | '÷' | 'ø' | 'ù' | 'ú' | 'û' | 'ü' | 'ý' | 'þ' | 'ÿ') [(-\U0010ffff]))> */ nil, - /* 34 mlLiteralString <- <('\'' '\'' '\'' ('\'' '\'' '\'') Action21)> */ + /* 35 mlLiteralString <- <('\'' '\'' '\'' ('\'' '\'' '\'') Action22)> */ nil, - /* 35 mlLiteralBody <- <(!('\'' '\'' '\'') (mlLiteralChar / newline))*> */ + /* 36 mlLiteralBody <- <(!('\'' '\'' '\'') (mlLiteralChar / newline))*> */ nil, - /* 36 mlLiteralChar <- <('\t' / [ -\U0010ffff])> */ + /* 37 mlLiteralChar <- <('\t' / [ -\U0010ffff])> */ nil, - /* 37 hexdigit <- <((&('a' | 'b' | 'c' | 'd' | 'e' | 'f') [a-f]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F') [A-F]) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]))> */ + /* 38 hexdigit <- <((&('a' | 'b' | 'c' | 'd' | 'e' | 'f') [a-f]) | (&('A' | 'B' | 'C' | 'D' | 'E' | 'F') [A-F]) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') [0-9]))> */ func() bool { - position267, tokenIndex267 := position, tokenIndex + position272, tokenIndex272 := position, tokenIndex { - position268 := position + position273 := position { switch buffer[position] { case 'a', 'b', 'c', 'd', 'e', 'f': if c := buffer[position]; c < rune('a') || c > rune('f') { - goto l267 + goto l272 } position++ break case 'A', 'B', 'C', 'D', 'E', 'F': if c := buffer[position]; c < rune('A') || c > rune('F') { - goto l267 + goto l272 } position++ break default: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l267 + goto l272 } position++ break } } - add(rulehexdigit, position268) + add(rulehexdigit, position273) } return true - l267: - position, tokenIndex = position267, tokenIndex267 + l272: + position, tokenIndex = position272, tokenIndex272 return false }, - /* 38 hexQuad <- <(hexdigit hexdigit hexdigit hexdigit)> */ + /* 39 hexQuad <- <(hexdigit hexdigit hexdigit hexdigit)> */ func() bool { - position270, tokenIndex270 := position, tokenIndex + position275, tokenIndex275 := position, tokenIndex { - position271 := position + position276 := position if !_rules[rulehexdigit]() { - goto l270 + goto l275 } if !_rules[rulehexdigit]() { - goto l270 + goto l275 } if !_rules[rulehexdigit]() { - goto l270 + goto l275 } if !_rules[rulehexdigit]() { - goto l270 + goto l275 } - add(rulehexQuad, position271) + add(rulehexQuad, position276) } return true - l270: - position, tokenIndex = position270, tokenIndex270 + l275: + position, tokenIndex = position275, tokenIndex275 return false }, - /* 39 boolean <- <(('t' 'r' 'u' 'e') / ('f' 'a' 'l' 's' 'e'))> */ + /* 40 boolean <- <(('t' 'r' 'u' 'e') / ('f' 'a' 'l' 's' 'e'))> */ nil, - /* 40 dateFullYear <- */ + /* 41 dateFullYear <- */ nil, - /* 41 dateMonth <- */ + /* 42 dateMonth <- */ nil, - /* 42 dateMDay <- */ + /* 43 dateMDay <- */ nil, - /* 43 timeHour <- */ + /* 44 timeHour <- */ func() bool { - position276, tokenIndex276 := position, tokenIndex + position281, tokenIndex281 := position, tokenIndex { - position277 := position + position282 := position if !_rules[ruledigitDual]() { - goto l276 + goto l281 } - add(ruletimeHour, position277) + add(ruletimeHour, position282) } return true - l276: - position, tokenIndex = position276, tokenIndex276 + l281: + position, tokenIndex = position281, tokenIndex281 return false }, - /* 44 timeMinute <- */ + /* 45 timeMinute <- */ func() bool { - position278, tokenIndex278 := position, tokenIndex + position283, tokenIndex283 := position, tokenIndex { - position279 := position + position284 := position if !_rules[ruledigitDual]() { - goto l278 + goto l283 } - add(ruletimeMinute, position279) + add(ruletimeMinute, position284) } return true - l278: - position, tokenIndex = position278, tokenIndex278 + l283: + position, tokenIndex = position283, tokenIndex283 return false }, - /* 45 timeSecond <- */ + /* 46 timeSecond <- */ nil, - /* 46 timeSecfrac <- <('.' digit+)> */ + /* 47 timeSecfrac <- <('.' digit+)> */ nil, - /* 47 timeNumoffset <- <(('-' / '+') timeHour ':' timeMinute)> */ + /* 48 timeNumoffset <- <(('-' / '+') timeHour ':' timeMinute)> */ nil, - /* 48 timeOffset <- <('Z' / timeNumoffset)> */ + /* 49 timeOffset <- <('Z' / timeNumoffset)> */ nil, - /* 49 partialTime <- <(timeHour ':' timeMinute ':' timeSecond timeSecfrac?)> */ + /* 50 partialTime <- <(timeHour ':' timeMinute ':' timeSecond timeSecfrac?)> */ func() bool { - position284, tokenIndex284 := position, tokenIndex + position289, tokenIndex289 := position, tokenIndex { - position285 := position + position290 := position if !_rules[ruletimeHour]() { - goto l284 + goto l289 } if buffer[position] != rune(':') { - goto l284 + goto l289 } position++ if !_rules[ruletimeMinute]() { - goto l284 + goto l289 } if buffer[position] != rune(':') { - goto l284 + goto l289 } position++ { - position286 := position + position291 := position if !_rules[ruledigitDual]() { - goto l284 + goto l289 } - add(ruletimeSecond, position286) + add(ruletimeSecond, position291) } { - position287, tokenIndex287 := position, tokenIndex + position292, tokenIndex292 := position, tokenIndex { - position289 := position + position294 := position if buffer[position] != rune('.') { - goto l287 + goto l292 } position++ if !_rules[ruledigit]() { - goto l287 + goto l292 } - l290: + l295: { - position291, tokenIndex291 := position, tokenIndex + position296, tokenIndex296 := position, tokenIndex if !_rules[ruledigit]() { - goto l291 + goto l296 } - goto l290 - l291: - position, tokenIndex = position291, tokenIndex291 + goto l295 + l296: + position, tokenIndex = position296, tokenIndex296 } - add(ruletimeSecfrac, position289) + add(ruletimeSecfrac, position294) } - goto l288 - l287: - position, tokenIndex = position287, tokenIndex287 + goto l293 + l292: + position, tokenIndex = position292, tokenIndex292 } - l288: - add(rulepartialTime, position285) + l293: + add(rulepartialTime, position290) } return true - l284: - position, tokenIndex = position284, tokenIndex284 + l289: + position, tokenIndex = position289, tokenIndex289 return false }, - /* 50 fullDate <- <(dateFullYear '-' dateMonth '-' dateMDay)> */ + /* 51 fullDate <- <(dateFullYear '-' dateMonth '-' dateMDay)> */ nil, - /* 51 fullTime <- <(partialTime timeOffset)> */ + /* 52 fullTime <- <(partialTime timeOffset?)> */ nil, - /* 52 datetime <- <((fullDate ('T' fullTime)?) / partialTime)> */ + /* 53 datetime <- <((fullDate ('T' fullTime)?) / partialTime)> */ nil, - /* 53 digit <- <[0-9]> */ + /* 54 digit <- <[0-9]> */ func() bool { - position295, tokenIndex295 := position, tokenIndex + position300, tokenIndex300 := position, tokenIndex { - position296 := position + position301 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l295 + goto l300 } position++ - add(ruledigit, position296) + add(ruledigit, position301) } return true - l295: - position, tokenIndex = position295, tokenIndex295 + l300: + position, tokenIndex = position300, tokenIndex300 return false }, - /* 54 digitDual <- <(digit digit)> */ - func() bool { - position297, tokenIndex297 := position, tokenIndex - { - position298 := position - if !_rules[ruledigit]() { - goto l297 - } - if !_rules[ruledigit]() { - goto l297 - } - add(ruledigitDual, position298) - } - return true - l297: - position, tokenIndex = position297, tokenIndex297 - return false - }, - /* 55 digitQuad <- <(digitDual digitDual)> */ - nil, - /* 56 array <- <('[' Action22 wsnl arrayValues? wsnl ']')> */ - nil, - /* 57 arrayValues <- <(val Action23 (wsnl comment? wsnl arraySep wsnl comment? wsnl val Action24)* wsnl arraySep? wsnl comment?)> */ - nil, - /* 58 arraySep <- <','> */ + /* 55 digitDual <- <(digit digit)> */ func() bool { position302, tokenIndex302 := position, tokenIndex { position303 := position - if buffer[position] != rune(',') { + if !_rules[ruledigit]() { goto l302 } - position++ - add(rulearraySep, position303) + if !_rules[ruledigit]() { + goto l302 + } + add(ruledigitDual, position303) } return true l302: position, tokenIndex = position302, tokenIndex302 return false }, - /* 60 Action0 <- <{ _ = buffer }> */ + /* 56 digitQuad <- <(digitDual digitDual)> */ + nil, + /* 57 array <- <('[' Action23 wsnl arrayValues? wsnl ']')> */ + nil, + /* 58 arrayValues <- <(val Action24 (wsnl comment? wsnl arraySep wsnl comment? wsnl val Action25)* wsnl arraySep? wsnl comment?)> */ + nil, + /* 59 arraySep <- <','> */ + func() bool { + position307, tokenIndex307 := position, tokenIndex + { + position308 := position + if buffer[position] != rune(',') { + goto l307 + } + position++ + add(rulearraySep, position308) + } + return true + l307: + position, tokenIndex = position307, tokenIndex307 + return false + }, + /* 61 Action0 <- <{ _ = buffer }> */ nil, nil, - /* 62 Action1 <- <{ p.SetTableString(begin, end) }> */ + /* 63 Action1 <- <{ p.SetTableString(begin, end) }> */ nil, - /* 63 Action2 <- <{ p.AddLineCount(end - begin) }> */ + /* 64 Action2 <- <{ p.AddLineCount(end - begin) }> */ nil, - /* 64 Action3 <- <{ p.AddLineCount(end - begin) }> */ + /* 65 Action3 <- <{ p.AddLineCount(end - begin) }> */ nil, - /* 65 Action4 <- <{ p.AddKeyValue() }> */ + /* 66 Action4 <- <{ p.AddKeyValue() }> */ nil, - /* 66 Action5 <- <{ p.SetKey(p.buffer, begin, end) }> */ + /* 67 Action5 <- <{ p.SetKey(p.buffer, begin, end) }> */ nil, - /* 67 Action6 <- <{ p.SetKey(p.buffer, begin-1, end+1) }> */ + /* 68 Action6 <- <{ p.SetKey(p.buffer, begin, end) }> */ nil, - /* 68 Action7 <- <{ p.SetTime(begin, end) }> */ + /* 69 Action7 <- <{ p.SetTime(begin, end) }> */ nil, - /* 69 Action8 <- <{ p.SetFloat64(begin, end) }> */ + /* 70 Action8 <- <{ p.SetFloat64(begin, end) }> */ nil, - /* 70 Action9 <- <{ p.SetInt64(begin, end) }> */ + /* 71 Action9 <- <{ p.SetInt64(begin, end) }> */ nil, - /* 71 Action10 <- <{ p.SetString(begin, end) }> */ + /* 72 Action10 <- <{ p.SetString(begin, end) }> */ nil, - /* 72 Action11 <- <{ p.SetBool(begin, end) }> */ + /* 73 Action11 <- <{ p.SetBool(begin, end) }> */ nil, - /* 73 Action12 <- <{ p.SetArray(begin, end) }> */ + /* 74 Action12 <- <{ p.SetArray(begin, end) }> */ nil, - /* 74 Action13 <- <{ p.SetTable(p.buffer, begin, end) }> */ + /* 75 Action13 <- <{ p.SetTable(p.buffer, begin, end) }> */ nil, - /* 75 Action14 <- <{ p.SetArrayTable(p.buffer, begin, end) }> */ + /* 76 Action14 <- <{ p.SetArrayTable(p.buffer, begin, end) }> */ nil, - /* 76 Action15 <- <{ p.StartInlineTable() }> */ + /* 77 Action15 <- <{ p.StartInlineTable() }> */ nil, - /* 77 Action16 <- <{ p.EndInlineTable() }> */ + /* 78 Action16 <- <{ p.EndInlineTable() }> */ nil, - /* 78 Action17 <- <{ p.SetBasicString(p.buffer, begin, end) }> */ + /* 79 Action17 <- <{ p.AddTableKey() }> */ nil, - /* 79 Action18 <- <{ p.SetMultilineString() }> */ + /* 80 Action18 <- <{ p.SetBasicString(p.buffer, begin, end) }> */ nil, - /* 80 Action19 <- <{ p.AddMultilineBasicBody(p.buffer, begin, end) }> */ + /* 81 Action19 <- <{ p.SetMultilineString() }> */ nil, - /* 81 Action20 <- <{ p.SetLiteralString(p.buffer, begin, end) }> */ + /* 82 Action20 <- <{ p.AddMultilineBasicBody(p.buffer, begin, end) }> */ nil, - /* 82 Action21 <- <{ p.SetMultilineLiteralString(p.buffer, begin, end) }> */ + /* 83 Action21 <- <{ p.SetLiteralString(p.buffer, begin, end) }> */ nil, - /* 83 Action22 <- <{ p.StartArray() }> */ + /* 84 Action22 <- <{ p.SetMultilineLiteralString(p.buffer, begin, end) }> */ nil, - /* 84 Action23 <- <{ p.AddArrayVal() }> */ + /* 85 Action23 <- <{ p.StartArray() }> */ nil, - /* 85 Action24 <- <{ p.AddArrayVal() }> */ + /* 86 Action24 <- <{ p.AddArrayVal() }> */ + nil, + /* 87 Action25 <- <{ p.AddArrayVal() }> */ nil, } p.rules = _rules diff --git a/vendor/vendor.json b/vendor/vendor.json index 80eb8989e4..e5b2d579d4 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -292,10 +292,10 @@ "revisionTime": "2017-07-13T18:40:44Z" }, { - "checksumSHA1": "2gmvVTDCks8cPhpmyDlvm0sbrXE=", + "checksumSHA1": "FYM/8R2CqS6PSNAoKl6X5gNJ20A=", "path": "github.com/naoina/toml", - "revision": "ac014c6b6502388d89a85552b7208b8da7cfe104", - "revisionTime": "2017-04-10T21:57:17Z" + "revision": "9fafd69674167c06933b1787ae235618431ce87f", + "revisionTime": "2017-09-18T21:04:37Z" }, { "checksumSHA1": "xZBlSMT5o/A+EDOro6KbfHZwSNc=",