timeout option

cmd/crawley/main.go

@@ -20,10 +20,11 @@ import (
 )
 
 const (
-	appName      = "Crawley"
-	appHelp      = "the unix-way web crawler"
-	appSite      = "https://github.com/s0rg/crawley"
-	defaultDelay = 150 * time.Millisecond
+	appName        = "Crawley"
+	appHelp        = "the unix-way web crawler"
+	appSite        = "https://github.com/s0rg/crawley"
+	defaultDelay   = 150 * time.Millisecond
+	defaultTimeout = 5 * time.Second
 )
 
 // build-time values.
@@ -43,6 +44,7 @@ var (
 	fDirsPolicy, fProxyAuth string
 	fRobotsPolicy, fUA      string
 	fDelay                  time.Duration
+	fTimeout                time.Duration
 	cookies, headers        values.Smart
 	tags, ignored           values.List
 )
@@ -164,6 +166,7 @@ func initOptions() (rv []crawler.Option, err error) {
 		crawler.WithTagsFilter(tags.Values),
 		crawler.WithIgnored(ignored.Values),
 		crawler.WithProxyAuth(fProxyAuth),
+		crawler.WithTimeout(fTimeout),
 	}
 
 	return rv, nil
@@ -198,6 +201,7 @@ func setupFlags() {
 	flag.StringVar(&fProxyAuth, "proxy-auth", "", "credentials for proxy: user:password")
 
 	flag.DurationVar(&fDelay, "delay", defaultDelay, "per-request delay (0 - disable)")
+	flag.DurationVar(&fTimeout, "timeout", defaultTimeout, "request timeout (min: 1 second, max: 10 minutes)")
 
 	flag.Usage = usage
 }

pkg/client/http.go

@@ -9,11 +9,7 @@ import (
 	"time"
 )
 
-const (
-	idleTimeout = 5 * time.Second
-	dialTimeout = 5 * time.Second
-	reqTimeout  = 10 * time.Second
-)
+const transportTimeout = 10 * time.Second
 
 // HTTP holds pre-configured http.Client.
 type HTTP struct {
@@ -29,24 +25,25 @@ func New(
 	conns int,
 	skipSSL bool,
 	headers, cookies []string,
+	timeout time.Duration,
 ) (h *HTTP) {
 	transport := &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		Dial: (&net.Dialer{
-			Timeout: dialTimeout,
+			Timeout: transportTimeout,
 		}).Dial,
 		TLSClientConfig: &tls.Config{
 			InsecureSkipVerify: skipSSL,
 		},
-		IdleConnTimeout:     idleTimeout,
-		TLSHandshakeTimeout: dialTimeout,
+		IdleConnTimeout:     transportTimeout,
+		TLSHandshakeTimeout: transportTimeout,
 		MaxConnsPerHost:     conns,
 		MaxIdleConns:        conns,
 		MaxIdleConnsPerHost: conns,
 	}
 
 	client := &http.Client{
-		Timeout:   reqTimeout,
+		Timeout:   timeout,
 		Transport: transport,
 	}
 

pkg/client/http_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"testing"
+	"time"
 )
 
 const (
@@ -15,7 +16,7 @@ const (
 func TestHTTPGetOK(t *testing.T) {
 	t.Parallel()
 
-	c := New(ua, 1, false, []string{"FOO: BAR"}, []string{"NAME=VALUE"})
+	c := New(ua, 1, false, []string{"FOO: BAR"}, []string{"NAME=VALUE"}, time.Minute)
 
 	const (
 		body   = "test-body"
@@ -71,7 +72,7 @@ func TestHTTPGetOK(t *testing.T) {
 func TestHTTPGetERR(t *testing.T) {
 	t.Parallel()
 
-	c := New("", 1, false, []string{}, []string{})
+	c := New("", 1, false, []string{}, []string{}, time.Second)
 
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusNotFound)
@@ -97,7 +98,7 @@ func TestHTTPGetERR(t *testing.T) {
 func TestHTTPHeadOK(t *testing.T) {
 	t.Parallel()
 
-	c := New(ua, 1, false, []string{}, []string{})
+	c := New(ua, 1, false, []string{}, []string{}, time.Second)
 
 	const (
 		key = "x-some-key"
@@ -132,7 +133,7 @@ func TestHTTPHeadOK(t *testing.T) {
 func TestHTTPHeadERR(t *testing.T) {
 	t.Parallel()
 
-	c := New("", 1, false, []string{}, []string{})
+	c := New("", 1, false, []string{}, []string{}, time.Second)
 
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
 		w.WriteHeader(http.StatusInternalServerError)

pkg/crawler/config.go

@@ -10,6 +10,8 @@ const (
 	minWorkers = 1
 	maxWorkers = 64
 	minDelay   = time.Duration(0)
+	minTimeout = time.Second
+	maxTimeout = time.Minute * 10
 )
 
 type config struct {
@@ -27,6 +29,7 @@ type config struct {
 	Brute      bool
 	NoHEAD     bool
 	ScanJS     bool
+	Timeout    time.Duration
 }
 
 func (c *config) validate() {
@@ -37,6 +40,13 @@ func (c *config) validate() {
 		c.Workers = maxWorkers
 	}
 
+	switch {
+	case c.Timeout < minTimeout:
+		c.Timeout = minTimeout
+	case c.Timeout > maxTimeout:
+		c.Timeout = maxTimeout
+	}
+
 	if c.Delay < minDelay {
 		c.Delay = minDelay
 	}
@@ -49,7 +59,7 @@ func (c *config) validate() {
 func (c *config) String() (rv string) {
 	var sb strings.Builder
 
-	_, _ = sb.WriteString(fmt.Sprintf("workers: %d depth: %d", c.Workers, c.Depth))
+	_, _ = sb.WriteString(fmt.Sprintf("workers: %d depth: %d timeout: %s", c.Workers, c.Depth, c.Timeout))
 
 	if c.Brute {
 		_, _ = sb.WriteString(" brute: on")

pkg/crawler/config_test.go

@@ -19,6 +19,7 @@ func TestValidate(t *testing.T) {
 
 	c.Workers = 1000000
 	c.Delay = time.Duration(-100)
+	c.Timeout = time.Hour
 	c.Depth = -5
 
 	c.validate()
@@ -34,6 +35,10 @@ func TestValidate(t *testing.T) {
 	if c.Depth != -1 {
 		t.Error("non empty - bad depth")
 	}
+
+	if c.Timeout != maxTimeout {
+		t.Error("non empty - bad timeout")
+	}
 }
 
 func TestOptions(t *testing.T) {
@@ -42,6 +47,7 @@ func TestOptions(t *testing.T) {
 		rp      = RobotsRespect
 		dp      = DirsOnly
 		delay   = time.Hour
+		timeout = time.Minute * 5
 		workers = 13
 		depth   = 666
 		fbool   = true
@@ -69,6 +75,7 @@ func TestOptions(t *testing.T) {
 		WithTagsFilter([]string{"a", "form"}),
 		WithScanJS(fbool),
 		WithIgnored([]string{"logout"}),
+		WithTimeout(timeout),
 	}
 
 	c := &config{}
@@ -134,6 +141,10 @@ func TestOptions(t *testing.T) {
 	if len(c.Ignored) != 1 {
 		t.Error("unexpected ignored size")
 	}
+
+	if c.Timeout != timeout {
+		t.Error("bad timeout")
+	}
 }
 
 func TestString(t *testing.T) {

pkg/crawler/crawler.go

@@ -26,11 +26,8 @@ type crawlClient interface {
 }
 
 const (
-	chMult = 256
-
-	chanTimeout   = 100 * time.Millisecond
-	crawlTimeout  = 5 * time.Second
-	robotsTimeout = 3 * time.Second
+	chMult    = 256
+	chTimeout = 100 * time.Millisecond
 )
 
 type taskFlag byte
@@ -102,6 +99,7 @@ func (c *Crawler) Run(uri string, fn func(string)) (err error) {
 		c.cfg.SkipSSL,
 		c.cfg.Headers,
 		c.cfg.Cookies,
+		c.cfg.Timeout,
 	)
 	c.initRobots(base, web)
 
@@ -159,7 +157,7 @@ func (c *Crawler) emit(u string) {
 		return
 	}
 
-	t := time.NewTimer(chanTimeout)
+	t := time.NewTimer(chTimeout)
 	defer t.Stop()
 
 	select {
@@ -180,7 +178,7 @@ func (c *Crawler) crawl(base *url.URL, r *crawlResult) (yes bool) {
 		return
 	}
 
-	t := time.NewTimer(chanTimeout)
+	t := time.NewTimer(chTimeout)
 	defer t.Stop()
 
 	select {
@@ -208,7 +206,7 @@ func (c *Crawler) initRobots(host *url.URL, web crawlClient) {
 		return
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), robotsTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), c.cfg.Timeout)
 	defer cancel()
 
 	body, _, err := web.Get(ctx, robots.URL(host))
@@ -288,7 +286,7 @@ func (c *Crawler) linkHandler(a atom.Atom, s string) {
 		r.Flag = TaskCrawl
 	}
 
-	t := time.NewTimer(chanTimeout)
+	t := time.NewTimer(chTimeout)
 	defer t.Stop()
 
 	select {
@@ -344,7 +342,7 @@ func (c *Crawler) crawler(web crawlClient) {
 			time.Sleep(c.cfg.Delay)
 		}
 
-		ctx, cancel := context.WithTimeout(context.Background(), crawlTimeout)
+		ctx, cancel := context.WithTimeout(context.Background(), c.cfg.Timeout)
 		us := uri.String()
 
 		var parse bool

pkg/crawler/options.go

@@ -113,3 +113,10 @@ func WithProxyAuth(v string) Option {
 		}
 	}
 }
+
+// WithTimeout sets request timeout.
+func WithTimeout(v time.Duration) Option {
+	return func(c *config) {
+		c.Timeout = v
+	}
+}