Merge pull request #2 from s0rg/feature/depth-calc-improved

path.Depth speed + memory optimization

Author: s0rg

README.md

@@ -21,6 +21,7 @@ Crawls web pages and prints any link it can find.
 - scan depth (limited by starting host and path, by default - 0) can be configured
 - can crawl `robots.txt` rules and sitemaps
 - `brute` mode - scan html comments for urls (this can lead to bogus results)
+- make use of `HTTP_PROXY` / `HTTPS_PROXY` environment values
 
 # installation
 

pkg/crawler/crawl.go

@@ -286,7 +286,17 @@ func canCrawl(a, b *url.URL, d int) (yes bool) {
 		return
 	}
 
-	depth, found := path.Depth(a.EscapedPath(), b.EscapedPath())
+	var apath, bpath string
+
+	if apath = a.Path; apath == "" {
+		apath = "/"
+	}
+
+	if bpath = b.Path; bpath == "" {
+		bpath = "/"
+	}
+
+	depth, found := path.Depth(apath, bpath)
 	if !found {
 		return
 	}

pkg/crawler/crawl_test.go

@@ -30,6 +30,7 @@ func Test_canCrawl(t *testing.T) {
 	url0, _ := url.Parse("http://test/some")
 	url1, _ := url.Parse("http://test/some/path/even")
 	url2, _ := url.Parse("http://test/some/path/even/more")
+	url3, _ := url.Parse("http://test")
 
 	tests := []struct {
 		name    string
@@ -47,6 +48,7 @@ func Test_canCrawl(t *testing.T) {
 		{"url2-0-1", args{b: base, u: url0, d: -1}, false},
 		{"url2-1-1", args{b: base, u: url1, d: -1}, true},
 		{"url2-2-1", args{b: base, u: url2, d: -1}, true},
+		{"url3-3", args{b: base, u: url3, d: 0}, false},
 	}
 
 	for _, tt := range tests {
@@ -276,6 +278,7 @@ sitemap: http://other.host/sitemap.xml`
 	}
 
 	if len(resA) != 5 {
+		t.Log(resA)
 		t.Fatal("unexpected len for A")
 	}
 

pkg/path/depth.go

@@ -1,41 +1,38 @@
 package path
 
 import (
+	"path"
 	"strings"
 )
 
+const pathSep = '/'
+
+func isPathSep(r rune) (yes bool) {
+	return r == pathSep
+}
+
 // Depth calculates relative depth for `sub` to `base` resorces path.
 func Depth(base, sub string) (n int, ok bool) {
 	var (
-		bp = splitPath(base)
-		sp = splitPath(sub)
+		bn = path.Clean(base)
+		sn = path.Clean(sub)
 	)
 
-	if len(sp) < len(bp) {
+	if len(sn) <= len(bn) {
 		return
 	}
 
-	for i := 0; i < len(bp); i++ {
-		if bp[i] != sp[i] {
-			return
-		}
+	if !strings.HasPrefix(sn, bn) {
+		return
 	}
 
-	return len(sp) - len(bp), true
-}
-
-func splitPath(p string) (o []string) {
-	return dropSpaces(strings.Split(p, "/"))
-}
-
-func dropSpaces(s []string) (o []string) {
-	o = make([]string, 0, len(s))
+	fields := strings.FieldsFunc(sn[len(bn):], isPathSep)
 
-	for _, v := range s {
-		if v != "" {
-			o = append(o, v)
+	for i := 0; i < len(fields); i++ {
+		if fields[i] != "" {
+			n++
 		}
 	}
 
-	return o
+	return n, true
 }

pkg/path/depth_test.go

@@ -1,72 +1,9 @@
 package path
 
 import (
-	"reflect"
 	"testing"
 )
 
-func Test_dropSpaces(t *testing.T) {
-	type args struct {
-		s []string
-	}
-
-	tests := []struct {
-		name  string
-		args  args
-		wantO []string
-	}{
-		{"ones", args{s: []string{"", "1", ""}}, []string{"1"}},
-		{"twos", args{s: []string{"2", "", "2"}}, []string{"2", "2"}},
-		{"threes", args{s: []string{"", "", "3"}}, []string{"3"}},
-		{"empty", args{s: []string{"", "", ""}}, []string{}},
-	}
-
-	t.Parallel()
-
-	for _, tt := range tests {
-		tc := tt
-
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-
-			if gotO := dropSpaces(tc.args.s); !reflect.DeepEqual(gotO, tc.wantO) {
-				t.Errorf("dropSpaces() = %v, want %v", gotO, tc.wantO)
-			}
-		})
-	}
-}
-
-func Test_splitPath(t *testing.T) {
-	type args struct {
-		p string
-	}
-
-	tests := []struct {
-		name  string
-		args  args
-		wantO []string
-	}{
-		{"empty", args{p: "/"}, []string{}},
-		{"foo", args{p: "/foo"}, []string{"foo"}},
-		{"foo-bar", args{p: "/foo/bar"}, []string{"foo", "bar"}},
-		{"foo-bar-baz", args{p: "/foo/bar//baz"}, []string{"foo", "bar", "baz"}},
-	}
-
-	t.Parallel()
-
-	for _, tt := range tests {
-		tc := tt
-
-		t.Run(tc.name, func(t *testing.T) {
-			t.Parallel()
-
-			if gotO := splitPath(tc.args.p); !reflect.DeepEqual(gotO, tc.wantO) {
-				t.Errorf("splitPath() = %v, want %v", gotO, tc.wantO)
-			}
-		})
-	}
-}
-
 func Test_Depth(t *testing.T) {
 	type args struct {
 		base string
@@ -84,6 +21,7 @@ func Test_Depth(t *testing.T) {
 		{"c-bad", args{base: "/a/b", sub: "/c"}, 0, false},
 		{"b-ok", args{base: "/a", sub: "/a/b"}, 1, true},
 		{"c-ok", args{base: "/a", sub: "/a/b/c"}, 2, true},
+		{"d-bad", args{base: "/a/b/c", sub: "/d/b/c/a"}, 0, false},
 	}
 
 	t.Parallel()
@@ -104,3 +42,16 @@ func Test_Depth(t *testing.T) {
 		})
 	}
 }
+
+func Benchmark_Depth(b *testing.B) {
+	const (
+		x = "/some/rather/long/path"
+		y = "/some/rather/long/path/but/longer"
+	)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		_, _ = Depth(x, y)
+	}
+}