From b15960d933af64d8b8e2519432f02d9c8fd4e30e Mon Sep 17 00:00:00 2001 From: Fabio Barone Date: Thu, 31 Jan 2019 19:29:02 -0500 Subject: [PATCH] swarm: first version of HasChunk implementation --- cmd/swarm/config.go | 13 +++++++++++++ cmd/swarm/flags.go | 4 ++++ cmd/swarm/main.go | 2 ++ swarm/api/config.go | 2 ++ swarm/api/testapi.go | 31 ++++++++++++++++++++++++++++++ swarm/storage/common_test.go | 8 ++++++++ swarm/storage/ldbstore.go | 10 ++++++++++ swarm/storage/localstore.go | 7 +++++++ swarm/storage/localstore_test.go | 33 ++++++++++++++++++++++++++++++++ swarm/storage/memstore.go | 4 ++++ swarm/storage/netstore.go | 4 ++++ swarm/storage/types.go | 8 +++++++- swarm/swarm.go | 5 +++++ 13 files changed, 130 insertions(+), 1 deletion(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 0203a67984..d502a532e6 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -80,6 +80,7 @@ const ( SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE" + SWARM_ENV_DEBUG_API = "SWARM_DEBUG_API" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" GETH_ENV_DATADIR = "GETH_DATADIR" @@ -262,6 +263,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name) } + if ctx.GlobalIsSet(SwarmDebugAPIFlag.Name) { + currentConfig.DebugAPI = ctx.GlobalBool(SwarmDebugAPIFlag.Name) + } + return currentConfig } @@ -375,6 +380,14 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.BootnodeMode = bootnodeMode } + if debugAPIenable := os.Getenv(SWARM_ENV_DEBUG_API); debugAPIenable != "" { + debug, err := strconv.ParseBool(debugAPIenable) + if err != nil { + utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_DEBUG_API, err) + } + currentConfig.DebugAPI = debug + } + return currentConfig } diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 4c186cc310..0ccdb1873a 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -176,4 +176,8 @@ var ( Name: "user", Usage: "Indicates the user who updates the feed", } + SwarmDebugAPIFlag = cli.BoolFlag{ + Name: "debug-api", + Usage: "Make debug APIs available", + } ) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 53888b615f..ae029372fe 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -194,6 +194,8 @@ func init() { SwarmStorePath, SwarmStoreCapacity, SwarmStoreCacheCapacity, + // provide debug API endpoints + SwarmDebugAPIFlag, } rpcFlags := []cli.Flag{ utils.WSEnabledFlag, diff --git a/swarm/api/config.go b/swarm/api/config.go index 54dd67ba83..5c48664020 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -60,6 +60,7 @@ type Config struct { BzzKey string NodeID string NetworkID uint64 + DebugAPI bool SwapEnabled bool SyncEnabled bool SyncingSkipCheck bool @@ -90,6 +91,7 @@ func NewConfig() (c *Config) { EnsAPIs: nil, EnsRoot: ens.TestNetAddress, NetworkID: network.DefaultNetworkID, + DebugAPI: false, SwapEnabled: false, SyncEnabled: true, SyncingSkipCheck: false, diff --git a/swarm/api/testapi.go b/swarm/api/testapi.go index 6fec55f559..0d15da971c 100644 --- a/swarm/api/testapi.go +++ b/swarm/api/testapi.go @@ -17,7 +17,11 @@ package api import ( + "context" + + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" ) type Control struct { @@ -32,3 +36,30 @@ func NewControl(api *API, hive *network.Hive) *Control { func (c *Control) Hive() string { return c.hive.String() } + +type DebugAPI struct { + netStore *storage.NetStore +} + +func NewDebugAPI(nstore *storage.NetStore) *DebugAPI { + return &DebugAPI{ + netStore: nstore, + } +} + +func (dapi *DebugAPI) String() string { + return "debugapi" +} + +func (dapi *DebugAPI) HasChunk(chunkAddress storage.Address) bool { + return dapi.netStore.HasChunk(context.Background(), chunkAddress) +} + +func GetDebugAPIDesc(nstore *storage.NetStore) rpc.API { + return rpc.API{ + Namespace: "debugapi", + Version: "1.0", + Service: NewDebugAPI(nstore), + Public: false, + } +} diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index bcc29d8cc7..53ef94a568 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -266,6 +266,14 @@ func (m *MapChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { return chunk, nil } +func (m *MapChunkStore) HasChunk(ctx context.Context, ref Address) bool { + m.mu.RLock() + defer m.mu.RUnlock() + + _, has := m.chunks[ref.Hex()] + return has +} + func (m *MapChunkStore) Close() { } diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 635d334297..de6dffa234 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -969,6 +969,16 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) return s.get(addr) } +// HasChunk queries the underlying DB if a chunk with the given address is stored +// Returns true if the chunk is found, false if not +func (s *LDBStore) HasChunk(_ context.Context, addr Address) bool { + s.lock.RLock() + defer s.lock.RUnlock() + proximity := s.po(addr) + _, found := s.tryAccessIdx(addr, proximity) + return found +} + // TODO: To conform with other private methods of this object indices should not be updated func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { if s.closed { diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 956560902b..c86dc93411 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -132,6 +132,13 @@ func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error { return err } +// HasChunk queries the underlying DbStore if a chunk with the given address +// is being stored there. +// Returns true if it is stored, false if not +func (ls *LocalStore) HasChunk(ctx context.Context, addr Address) bool { + return ls.DbStore.HasChunk(ctx, addr) +} + // Get(chunk *Chunk) looks up a chunk in the local stores // This method is blocking until the chunk is retrieved // so additional timeout may be needed to wrap this call if diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 7a4162a475..6ce2ffe7dd 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -209,3 +209,36 @@ func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func()) return store, cleanup } + +func TestHasChunk(t *testing.T) { + ldbCap := defaultGCRatio + store, cleanup := setupLocalStore(t, ldbCap) + defer cleanup() + + nonStoredAddr := GenerateRandomChunk(128).Address() + + has := store.HasChunk(context.Background(), nonStoredAddr) + if has { + t.Fatal("Expected HasChunk() to return false, but returned true!") + } + + storeChunks := GenerateRandomChunks(128, 3) + for _, ch := range storeChunks { + err := store.Put(context.Background(), ch) + if err != nil { + t.Fatalf("Expected store to store chunk, but it failed: %v", err) + } + + has := store.HasChunk(context.Background(), ch.Address()) + if !has { + t.Fatal("Expected HasChunk() to return true, but returned false!") + } + } + + //let's be paranoic and test again that the non-existent chunk returns false + has = store.HasChunk(context.Background(), nonStoredAddr) + if has { + t.Fatal("Expected HasChunk() to return false, but returned true!") + } + +} diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 86e5813d1b..2fff8cb965 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -48,6 +48,10 @@ func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore) { } } +func (m *MemStore) HasChunk(_ context.Context, addr Address) bool { + return m.cache.Contains(addr) +} + func (m *MemStore) Get(_ context.Context, addr Address) (Chunk, error) { if m.disabled { return nil, ErrChunkNotFound diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 16bc48a9a2..2026ff0d71 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -158,6 +158,10 @@ func (n *NetStore) get(ctx context.Context, ref Address) (Chunk, func(context.Co return chunk, nil, nil } +func (n *NetStore) HasChunk(ctx context.Context, ref Address) bool { + return n.store.HasChunk(ctx, ref) +} + // getOrCreateFetcher attempts at retrieving an existing fetchers // if none exists, creates one and saves it in the fetchers cache // caller must hold the lock diff --git a/swarm/storage/types.go b/swarm/storage/types.go index d792352252..67cfd994f3 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -292,6 +292,7 @@ func (v *ContentAddressValidator) Validate(chunk Chunk) bool { type ChunkStore interface { Put(ctx context.Context, ch Chunk) (err error) Get(rctx context.Context, ref Address) (ch Chunk, err error) + HasChunk(rctx context.Context, ref Address) bool Close() } @@ -314,7 +315,12 @@ func (f *FakeChunkStore) Put(_ context.Context, ch Chunk) error { return nil } -// Gut doesn't store anything it is just here to implement ChunkStore +// HasChunk doesn't do anything it is just here to implement ChunkStore +func (f *FakeChunkStore) HasChunk(_ context.Context, ref Address) bool { + panic("FakeChunkStore doesn't support HasChunk") +} + +// Get doesn't store anything it is just here to implement ChunkStore func (f *FakeChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { panic("FakeChunkStore doesn't support Get") } diff --git a/swarm/swarm.go b/swarm/swarm.go index 9ece8bb669..f2f5bd8bca 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -514,6 +514,11 @@ func (self *Swarm) APIs() []rpc.API { apis = append(apis, self.ps.APIs()...) } + if self.config.DebugAPI { + log.Info("Running node with debug APIs attached") + apis = append(apis, api.GetDebugAPIDesc(self.netStore)) + } + return apis }