diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 9bb719637a85..c1d3af3e4aac 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -68,6 +68,7 @@ const ( SWARM_ENV_SWAP_API = "SWARM_SWAP_API" SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE" SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY" + SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK" SWARM_ENV_ENS_API = "SWARM_ENS_API" SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" SWARM_ENV_CORS = "SWARM_CORS" @@ -203,6 +204,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.SyncUpdateDelay = d } + if ctx.GlobalIsSet(SwarmDeliverySkipCheckFlag.Name) { + currentConfig.DeliverySkipCheck = true + } + currentConfig.SwapApi = ctx.GlobalString(SwarmSwapAPIFlag.Name) if currentConfig.SwapEnabled && currentConfig.SwapApi == "" { utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) @@ -284,6 +289,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { } } + if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" { + if skipCheck, err := strconv.ParseBool(v); err != nil { + currentConfig.DeliverySkipCheck = skipCheck + } + } + if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" { if d, err := time.ParseDuration(v); err != nil { currentConfig.SyncUpdateDelay = d diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index 9424ba9bf70e..636d2c0eda86 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -88,6 +88,7 @@ func TestConfigCmdLineOverrides(t *testing.T) { fmt.Sprintf("--%s", SwarmSyncDisabledFlag.Name), fmt.Sprintf("--%s", CorsStringFlag.Name), "*", fmt.Sprintf("--%s", SwarmAccountFlag.Name), account.Address.String(), + fmt.Sprintf("--%s", SwarmDeliverySkipCheckFlag.Name), fmt.Sprintf("--%s", EnsAPIFlag.Name), "", "--datadir", dir, "--ipcpath", conf.IPCPath, @@ -128,6 +129,10 @@ func TestConfigCmdLineOverrides(t *testing.T) { t.Fatal("Expected Sync to be disabled, but is true") } + if !info.DeliverySkipCheck { + t.Fatal("Expected DeliverySkipCheck to be enabled, but it is not") + } + if info.Cors != "*" { t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors) } @@ -148,6 +153,7 @@ func TestConfigFileOverrides(t *testing.T) { defaultConf := api.NewConfig() //change some values in order to test if they have been loaded defaultConf.SyncEnabled = false + defaultConf.DeliverySkipCheck = true defaultConf.NetworkId = 54 defaultConf.Port = httpPort defaultConf.DbCapacity = 9000000 @@ -222,6 +228,10 @@ func TestConfigFileOverrides(t *testing.T) { t.Fatal("Expected Sync to be disabled, but is true") } + if !info.DeliverySkipCheck { + t.Fatal("Expected DeliverySkipCheck to be enabled, but it is not") + } + if info.DbCapacity != 9000000 { t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId) } @@ -253,6 +263,7 @@ func TestConfigEnvVars(t *testing.T) { envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmNetworkIdFlag.EnvVar, "999")) envVars = append(envVars, fmt.Sprintf("%s=%s", CorsStringFlag.EnvVar, "*")) envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmSyncDisabledFlag.EnvVar, "true")) + envVars = append(envVars, fmt.Sprintf("%s=%s", SwarmDeliverySkipCheckFlag.EnvVar, "true")) dir, err := ioutil.TempDir("", "bzztest") if err != nil { @@ -333,6 +344,10 @@ func TestConfigEnvVars(t *testing.T) { t.Fatal("Expected Sync to be disabled, but is true") } + if !info.DeliverySkipCheck { + t.Fatal("Expected DeliverySkipCheck to be enabled, but it is not") + } + node.Shutdown() cmd.Process.Kill() } diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 6612983fd7e5..696d4f1c1fc0 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -107,6 +107,11 @@ var ( Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)", EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY, } + SwarmDeliverySkipCheckFlag = cli.BoolFlag{ + Name: "delivery-skip-check", + Usage: "Skip chunk delivery check (default false)", + EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK, + } EnsAPIFlag = cli.StringSliceFlag{ Name: "ens-api", Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", @@ -343,6 +348,7 @@ Remove corrupt entries from a local chunk database. SwarmSwapAPIFlag, SwarmSyncDisabledFlag, SwarmSyncUpdateDelay, + SwarmDeliverySkipCheckFlag, SwarmListenAddrFlag, SwarmPortFlag, SwarmAccountFlag, diff --git a/swarm/api/config.go b/swarm/api/config.go index 289f6cc434df..cc5ed6f25b4b 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -50,24 +50,25 @@ type Config struct { Swap *swap.SwapParams Pss *pss.PssParams //*network.SyncParams - Contract common.Address - EnsRoot common.Address - EnsAPIs []string - Path string - ListenAddr string - Port string - PublicKey string - BzzKey string - NodeID string - NetworkId uint64 - SwapEnabled bool - SyncEnabled bool - SyncUpdateDelay time.Duration - SwapApi string - Cors string - BzzAccount string - BootNodes string - privateKey *ecdsa.PrivateKey + Contract common.Address + EnsRoot common.Address + EnsAPIs []string + Path string + ListenAddr string + Port string + PublicKey string + BzzKey string + NodeID string + NetworkId uint64 + SwapEnabled bool + SyncEnabled bool + DeliverySkipCheck bool + SyncUpdateDelay time.Duration + SwapApi string + Cors string + BzzAccount string + BootNodes string + privateKey *ecdsa.PrivateKey } //create a default config with all parameters to set to defaults @@ -78,19 +79,20 @@ func NewConfig() (self *Config) { DPAParams: storage.NewDPAParams(), HiveParams: network.NewHiveParams(), //SyncParams: network.NewDefaultSyncParams(), - Swap: swap.NewDefaultSwapParams(), - Pss: pss.NewPssParams(), - ListenAddr: DefaultHTTPListenAddr, - Port: DefaultHTTPPort, - Path: node.DefaultDataDir(), - EnsAPIs: nil, - EnsRoot: ens.TestNetAddress, - NetworkId: network.NetworkID, - SwapEnabled: false, - SyncEnabled: true, - SyncUpdateDelay: 15 * time.Second, - SwapApi: "", - BootNodes: "", + Swap: swap.NewDefaultSwapParams(), + Pss: pss.NewPssParams(), + ListenAddr: DefaultHTTPListenAddr, + Port: DefaultHTTPPort, + Path: node.DefaultDataDir(), + EnsAPIs: nil, + EnsRoot: ens.TestNetAddress, + NetworkId: network.NetworkID, + SwapEnabled: false, + SyncEnabled: true, + DeliverySkipCheck: false, + SyncUpdateDelay: 15 * time.Second, + SwapApi: "", + BootNodes: "", } return diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index d607c17f8472..7446d2dd9974 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -413,3 +413,16 @@ func (s *testExternalServer) GetData([]byte) ([]byte, error) { } func (s *testExternalServer) Close() {} + +// Sets the global value defaultSkipCheck. +// It should be used in test function defer to reset the global value +// to the original value. +// +// defer setDefaultSkipCheck(defaultSkipCheck) +// defaultSkipCheck = skipCheck +// +// This works as defer function arguments evaluations are evaluated as ususal, +// but only the function body invocation is deferred. +func setDefaultSkipCheck(skipCheck bool) { + defaultSkipCheck = skipCheck +} diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 4dd12191610d..ec96b13b488c 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -69,17 +69,22 @@ func newIntervalsStreamerService(ctx *adapters.ServiceContext) (node.Service, er } func TestIntervals(t *testing.T) { - testIntervals(t, true, nil) - testIntervals(t, false, NewRange(9, 26)) - testIntervals(t, true, NewRange(9, 26)) + testIntervals(t, true, nil, false) + testIntervals(t, false, NewRange(9, 26), false) + testIntervals(t, true, NewRange(9, 26), false) + + testIntervals(t, true, nil, true) + testIntervals(t, false, NewRange(9, 26), true) + testIntervals(t, true, NewRange(9, 26), true) } -func testIntervals(t *testing.T, live bool, history *Range) { +func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { nodes := 2 chunkCount := dataChunkCount - skipCheck := false + defer setDefaultSkipCheck(defaultSkipCheck) defaultSkipCheck = skipCheck + toAddr = network.NewAddrFromNodeID conf := &streamTesting.RunConfig{ Adapter: *adapter, diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 4e443cbda457..1b19e0c2a3eb 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -45,6 +45,7 @@ func TestSyncerSimulation(t *testing.T) { } func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) { + defer setDefaultSkipCheck(defaultSkipCheck) defaultSkipCheck = skipCheck createStoreFunc = createTestLocalStorageFromSim registries = make(map[discover.NodeID]*TestRegistry) diff --git a/swarm/network_test.go b/swarm/network_test.go index 9f4092c909e0..083f936717bc 100644 --- a/swarm/network_test.go +++ b/swarm/network_test.go @@ -62,7 +62,7 @@ func TestSwarmNetwork(t *testing.T) { for _, tc := range []struct { name string steps []testSwarmNetworkStep - timeout time.Duration + options *testSwarmNetworkOptions disabled bool }{ { @@ -72,7 +72,21 @@ func TestSwarmNetwork(t *testing.T) { nodeCount: 10, }, }, - timeout: 45 * time.Second, + options: &testSwarmNetworkOptions{ + Timeout: 45 * time.Second, + }, + }, + { + name: "10_nodes_skip_check", + steps: []testSwarmNetworkStep{ + { + nodeCount: 10, + }, + }, + options: &testSwarmNetworkOptions{ + Timeout: 45 * time.Second, + SkipCheck: true, + }, }, { name: "100_nodes", @@ -81,7 +95,22 @@ func TestSwarmNetwork(t *testing.T) { nodeCount: 100, }, }, - timeout: 3 * time.Minute, + options: &testSwarmNetworkOptions{ + Timeout: 3 * time.Minute, + }, + disabled: !*longrunning, + }, + { + name: "100_nodes_skip_check", + steps: []testSwarmNetworkStep{ + { + nodeCount: 100, + }, + }, + options: &testSwarmNetworkOptions{ + Timeout: 3 * time.Minute, + SkipCheck: true, + }, disabled: !*longrunning, }, { @@ -97,7 +126,9 @@ func TestSwarmNetwork(t *testing.T) { nodeCount: 10, }, }, - timeout: 90 * time.Second, + options: &testSwarmNetworkOptions{ + Timeout: 90 * time.Second, + }, disabled: !*longrunning, }, { @@ -113,7 +144,9 @@ func TestSwarmNetwork(t *testing.T) { nodeCount: 3, }, }, - timeout: 90 * time.Second, + options: &testSwarmNetworkOptions{ + Timeout: 90 * time.Second, + }, disabled: !*longrunning, }, { @@ -129,7 +162,9 @@ func TestSwarmNetwork(t *testing.T) { nodeCount: 10, }, }, - timeout: 90 * time.Second, + options: &testSwarmNetworkOptions{ + Timeout: 90 * time.Second, + }, }, { name: "inc_dec_node_count", @@ -150,7 +185,34 @@ func TestSwarmNetwork(t *testing.T) { nodeCount: 4, }, }, - timeout: 5 * time.Minute, + options: &testSwarmNetworkOptions{ + Timeout: 5 * time.Minute, + }, + disabled: !*longrunning, + }, + { + name: "inc_dec_node_count_skip_check", + steps: []testSwarmNetworkStep{ + { + nodeCount: 3, + }, + { + nodeCount: 5, + }, + { + nodeCount: 25, + }, + { + nodeCount: 10, + }, + { + nodeCount: 4, + }, + }, + options: &testSwarmNetworkOptions{ + Timeout: 5 * time.Minute, + SkipCheck: true, + }, disabled: !*longrunning, }, } { @@ -158,7 +220,7 @@ func TestSwarmNetwork(t *testing.T) { continue } t.Run(tc.name, func(t *testing.T) { - testSwarmNetwork(t, tc.timeout, tc.steps...) + testSwarmNetwork(t, tc.options, tc.steps...) }) } } @@ -184,6 +246,13 @@ type check struct { nodeID discover.NodeID } +// testSwarmNetworkOptions contains optional parameters for running +// testSwarmNetwork. +type testSwarmNetworkOptions struct { + Timeout time.Duration + SkipCheck bool +} + // testSwarmNetwork is a helper function used for testing different // static and dynamic Swarm network simulations. // It is responsible for: @@ -191,17 +260,21 @@ type check struct { // - Uploading a unique file to every node on every step. // - May wait for Kademlia on every node to be healthy. // - Checking if a file is retrievable from all nodes. -func testSwarmNetwork(t *testing.T, timeout time.Duration, steps ...testSwarmNetworkStep) { +func testSwarmNetwork(t *testing.T, o *testSwarmNetworkOptions, steps ...testSwarmNetworkStep) { dir, err := ioutil.TempDir("", "swarm-network-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) + if o == nil { + o = new(testSwarmNetworkOptions) + } + ctx := context.Background() - if timeout > 0 { + if o.Timeout > 0 { var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, timeout) + ctx, cancel = context.WithTimeout(ctx, o.Timeout) defer cancel() } @@ -225,6 +298,8 @@ func testSwarmNetwork(t *testing.T, timeout time.Duration, steps ...testSwarmNet } config.Init(privkey) + config.DeliverySkipCheck = o.SkipCheck + s, err := NewSwarm(nil, nil, config, nil) if err != nil { return nil, err diff --git a/swarm/swarm.go b/swarm/swarm.go index 3ebd3dccfd1d..38ebb3ab962c 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -179,6 +179,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api. delivery := stream.NewDelivery(to, db) self.streamer = stream.NewRegistry(addr, delivery, db, stateStore, &stream.RegistryOptions{ + SkipCheck: config.DeliverySkipCheck, DoSync: config.SyncEnabled, DoRetrieve: true, SyncUpdateDelay: config.SyncUpdateDelay,