Skip to content
This repository has been archived by the owner on May 11, 2022. It is now read-only.

Commit

Permalink
issue #26: improve cralwer test
Browse files Browse the repository at this point in the history
  • Loading branch information
@kamilsk
kamilsk committed Apr 29, 2018
1 parent a7e5713 commit 39c1ff7
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 22 deletions.
1 change: 0 additions & 1 deletion cmd/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ cmd-help:
cmd-completion:
cd $(PKG_DIR) && go run $(GO_FILES) completion bash
cd $(PKG_DIR) && go run $(GO_FILES) completion zsh
cd $(PKG_DIR) && go run $(GO_FILES) completion something

.PHONY: cmd-urls
cmd-urls:
Expand Down
2 changes: 1 addition & 1 deletion http/availability/crawler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ func OnResponse(bus EventBus) func(*colly.Collector) {

func OnHTML(base *url.URL, bus EventBus) func(*colly.Collector) {
isPage := func(current *url.URL) bool {
return current.Hostname() == base.Hostname()
return current.Host == base.Host
}
return func(c *colly.Collector) {
c.OnHTML("a[href]", func(el *colly.HTMLElement) {
Expand Down
18 changes: 11 additions & 7 deletions http/availability/crawler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,18 @@ func TestCrawlerColly(t *testing.T) {
defer closer()

{
var errorEvents, responseEvents, walkEvents, problemEvents, unknownEvents int
var errorEvents, redirectEvents, responseEvents, walkEvents, problemEvents, unknownEvents int
wg, bus := &sync.WaitGroup{}, availability.NewReadableEventBus(8)
wg.Add(1)
go func() {
defer wg.Done()
for event := range bus {
switch event.(type) {
switch e := event.(type) {
case availability.ErrorEvent:
errorEvents++
if e.Redirect != "" {
redirectEvents++
}
case availability.ResponseEvent:
responseEvents++
case availability.WalkEvent:
Expand All @@ -39,12 +42,13 @@ func TestCrawlerColly(t *testing.T) {
Verbose: true,
Output: ioutil.Discard,
})
assert.NoError(t, crawler.Visit(site.URL, bus))
assert.NoError(t, crawler.Visit(site.URL+"/", bus))
wg.Wait()
assert.Equal(t, 11, errorEvents)
assert.Equal(t, 3, responseEvents)
assert.Equal(t, 42, walkEvents)
assert.Equal(t, 9, problemEvents)
assert.Equal(t, 30, errorEvents)
assert.Equal(t, 10, redirectEvents)
assert.Equal(t, 8, responseEvents)
assert.Equal(t, 39, walkEvents)
assert.Equal(t, 2, problemEvents)
assert.Empty(t, unknownEvents)
}

Expand Down
82 changes: 69 additions & 13 deletions http/availability/helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@ var echoCode = func(rw http.ResponseWriter, req *http.Request) error {

switch code {

// ok
case http.StatusOK:
fallthrough
case http.StatusCreated:
fallthrough
case http.StatusAccepted:
rw.WriteHeader(code)
return nil

// redirects
case http.StatusMovedPermanently:
fallthrough
Expand All @@ -45,6 +54,29 @@ var echoCode = func(rw http.ResponseWriter, req *http.Request) error {
http.Error(rw, http.StatusText(code), code)
return nil

// client errors
case http.StatusBadRequest:
fallthrough
case http.StatusUnauthorized:
fallthrough
case http.StatusForbidden:
fallthrough
case http.StatusNotFound:
fallthrough
// server errors
case http.StatusInternalServerError:
fallthrough
case http.StatusNotImplemented:
fallthrough
case http.StatusBadGateway:
fallthrough
case http.StatusServiceUnavailable:
fallthrough
// and others
case http.StatusMultipleChoices:
http.Error(rw, http.StatusText(code), code)
return nil

default:
return fmt.Errorf("can't handle request %q", req.URL.Path)
}
Expand All @@ -54,22 +86,46 @@ var echoLinks = func(site1, site2 string) func(http.ResponseWriter, *http.Reques
return func(rw http.ResponseWriter, req *http.Request) error {
if req.URL.Path == "/" {
links := []struct {
Href string
Href template.URL
Text string
}{
{Href: site1 + "/", Text: "site1"},
{Href: site1 + "/301", Text: "site1 - 301"},
{Href: site1 + "/302", Text: "site1 - 302"},
{Href: site1 + "/303", Text: "site1 - 303"},
{Href: site1 + "/307", Text: "site1 - 307"},
{Href: site1 + "/308", Text: "site1 - 308"},
{Href: template.URL(site1 + "/"), Text: "site1"},
{Href: template.URL(site1 + "/200"), Text: "site1 - 200"},
{Href: template.URL(site1 + "/201"), Text: "site1 - 201"},
{Href: template.URL(site1 + "/202"), Text: "site1 - 202"},
{Href: template.URL(site1 + "/301"), Text: "site1 - 301"},
{Href: template.URL(site1 + "/302"), Text: "site1 - 302"},
{Href: template.URL(site1 + "/303"), Text: "site1 - 303"},
{Href: template.URL(site1 + "/307"), Text: "site1 - 307"},
{Href: template.URL(site1 + "/308"), Text: "site1 - 308"},
{Href: template.URL(site1 + "/400"), Text: "site1 - 400"},
{Href: template.URL(site1 + "/401"), Text: "site1 - 401"},
{Href: template.URL(site1 + "/403"), Text: "site1 - 403"},
{Href: template.URL(site1 + "/404"), Text: "site1 - 404"},
{Href: template.URL(site1 + "/500"), Text: "site1 - 500"},
{Href: template.URL(site1 + "/501"), Text: "site1 - 501"},
{Href: template.URL(site1 + "/502"), Text: "site1 - 502"},
{Href: template.URL(site1 + "/503"), Text: "site1 - 503"},
{Href: template.URL(site1 + "/300"), Text: "site1 - 300"},

{Href: site2 + "/", Text: "site2"},
{Href: site2 + "/301", Text: "site2 - 301"},
{Href: site2 + "/302", Text: "site2 - 302"},
{Href: site2 + "/303", Text: "site2 - 303"},
{Href: site2 + "/307", Text: "site2 - 307"},
{Href: site2 + "/308", Text: "site2 - 308"},
{Href: template.URL(site2 + "/"), Text: "site2"},
{Href: template.URL(site2 + "/200"), Text: "site2 - 200"},
{Href: template.URL(site2 + "/201"), Text: "site2 - 201"},
{Href: template.URL(site2 + "/202"), Text: "site2 - 202"},
{Href: template.URL(site2 + "/301"), Text: "site2 - 301"},
{Href: template.URL(site2 + "/302"), Text: "site2 - 302"},
{Href: template.URL(site2 + "/303"), Text: "site2 - 303"},
{Href: template.URL(site2 + "/307"), Text: "site2 - 307"},
{Href: template.URL(site2 + "/308"), Text: "site2 - 308"},
{Href: template.URL(site2 + "/400"), Text: "site2 - 400"},
{Href: template.URL(site2 + "/401"), Text: "site2 - 401"},
{Href: template.URL(site2 + "/403"), Text: "site2 - 403"},
{Href: template.URL(site2 + "/404"), Text: "site2 - 404"},
{Href: template.URL(site2 + "/500"), Text: "site2 - 500"},
{Href: template.URL(site2 + "/501"), Text: "site2 - 501"},
{Href: template.URL(site2 + "/502"), Text: "site2 - 502"},
{Href: template.URL(site2 + "/503"), Text: "site2 - 503"},
{Href: template.URL(site2 + "/300"), Text: "site2 - 300"},

//issue#34, fixes issue#30
{Href: "#anchor", Text: "anchor"},
Expand Down

0 comments on commit 39c1ff7

Please sign in to comment.