Merge pull request #5 from discoverygarden/cidr-support

SUP-6577: Add cidr support

.github/workflows/ci.yaml

@@ -0,0 +1,26 @@
+name: CI
+# yamllint thinks the `on` key is being turned into `true`
+# yamllint disable-line rule:truthy
+on: [push]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go-version: [ '1.19', '1.20', '1.21.x' ]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Run tests
+        run: go test -v .
+  lint-yaml:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Lint YAML
+        run: yamllint .

.github/workflows/semver.yaml

@@ -0,0 +1,20 @@
+name: Auto Semver
+# yamllint thinks the `on` key is being turned into `true`
+# yamllint disable-line rule:truthy
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+jobs:
+  update:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Run Auto Semver
+        uses: discoverygarden/auto-semver@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

.yamllint.yaml

@@ -0,0 +1,6 @@
+extends: default
+rules:
+  document-start: disable
+  line-length: disable
+  brackets:
+    max-spaces-inside: 1

README.md

@@ -23,3 +23,7 @@ spec:
 ## Blocklist
 
 The blocklists should be acccessible via http/s and be a plain text list of IP address or useragents.
+
+## Testing
+
+Running `go test` will run a set of unit tests. Running `docker compose up` will start an end to end testing environment where `allowed-*` containers should be able to make requests, while `blocked-*` containers should fail.

botblocker.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io"
 
 	"net/http"
 	"net/netip"
@@ -31,60 +32,103 @@ func CreateConfig() *Config {
 type BotBlocker struct {
 	next               http.Handler
 	name               string
-	ipBlocklist        []netip.Addr
+	prefixBlocklist    []netip.Prefix
 	userAgentBlockList []string
 	lastUpdated        time.Time
 	Config
 }
 
-func (b *BotBlocker) Update() error {
+func (b *BotBlocker) update() error {
 	startTime := time.Now()
-	err := b.UpdateIps()
+	err := b.updateIps()
 	if err != nil {
-		return fmt.Errorf("failed to update IP blocklists: %w", err)
+		return fmt.Errorf("failed to update CIDR blocklists: %w", err)
 	}
-	err = b.UpdateUserAgents()
+	err = b.updateUserAgents()
 	if err != nil {
-		return fmt.Errorf("failed to update IP blocklists: %w", err)
+		return fmt.Errorf("failed to update user agent blocklists: %w", err)
 	}
 
 	b.lastUpdated = time.Now()
-	duration := time.Now().Sub(startTime)
-	log.Info("Updated block lists. Blocked IPs: ", len(b.ipBlocklist), " Duration: ", duration)
+	duration := time.Since(startTime)
+	log.Info("Updated block lists. Blocked CIDRs: ", len(b.prefixBlocklist), " Duration: ", duration)
 	return nil
 }
 
-func (b *BotBlocker) UpdateIps() error {
-	ipBlockList := make([]netip.Addr, 0)
+func (b *BotBlocker) updateIps() error {
+	prefixBlockList := make([]netip.Prefix, 0)
 
-	log.Info("Updating IP blocklist")
+	log.Info("Updating CIDR blocklist")
 	for _, url := range b.IpBlocklistUrls {
 		resp, err := http.Get(url)
 		if err != nil {
-			return fmt.Errorf("failed fetch IP list: %w", err)
+			return fmt.Errorf("failed fetch CIDR list: %w", err)
 		}
 		if resp.StatusCode > 299 {
-			return fmt.Errorf("failed fetch IP list: received a %v from %v", resp.Status, url)
+			return fmt.Errorf("failed to fetch CIDR list: received a %v from %v", resp.Status, url)
 		}
 
-		defer resp.Body.Close()
-		scanner := bufio.NewScanner(resp.Body)
-		for scanner.Scan() {
-			addrStr := scanner.Text()
-			addr, err := netip.ParseAddr(addrStr)
+		prefixes, err := readPrefixes(resp.Body)
+		if err != nil {
+			return fmt.Errorf("failed to update CIDRs: %e", err)
+		}
+		prefixBlockList = append(prefixBlockList, prefixes...)
+	}
+
+	b.prefixBlocklist = prefixBlockList
+
+	return nil
+}
+
+func readPrefixes(prefixReader io.ReadCloser) ([]netip.Prefix, error) {
+	prefixes := make([]netip.Prefix, 0)
+	defer prefixReader.Close()
+	scanner := bufio.NewScanner(prefixReader)
+	for scanner.Scan() {
+		entry := strings.TrimSpace(scanner.Text())
+		var prefix netip.Prefix
+		if strings.Contains(entry, "/") {
+			var err error
+			prefix, err = netip.ParsePrefix(entry)
 			if err != nil {
-				return fmt.Errorf("failed to parse IP address: %w", err)
+				return []netip.Prefix{}, err
+			}
+		} else {
+			addr, err := netip.ParseAddr(entry)
+			if err != nil {
+				return []netip.Prefix{}, err
+			}
+			var bits int
+			if addr.Is4() {
+				bits = 32
+			} else {
+				bits = 128
+			}
+			prefix, err = addr.Prefix(bits)
+			if err != nil {
+				return []netip.Prefix{}, err
 			}
-			ipBlockList = append(ipBlockList, addr)
 		}
+		prefixes = append(prefixes, prefix)
 	}
 
-	b.ipBlocklist = ipBlockList
+	return prefixes, nil
+}
 
-	return nil
+func readUserAgents(userAgentReader io.ReadCloser) ([]string, error) {
+	userAgents := make([]string, 0)
+
+	defer userAgentReader.Close()
+	scanner := bufio.NewScanner(userAgentReader)
+	for scanner.Scan() {
+		agent := strings.ToLower(strings.TrimSpace(scanner.Text()))
+		userAgents = append(userAgents, agent)
+	}
+
+	return userAgents, nil
 }
 
-func (b *BotBlocker) UpdateUserAgents() error {
+func (b *BotBlocker) updateUserAgents() error {
 	userAgentBlockList := make([]string, 0)
 
 	log.Info("Updating user agent blocklist")
@@ -97,12 +141,11 @@ func (b *BotBlocker) UpdateUserAgents() error {
 			return fmt.Errorf("failed fetch useragent list: received a %v from %v", resp.Status, url)
 		}
 
-		defer resp.Body.Close()
-		scanner := bufio.NewScanner(resp.Body)
-		for scanner.Scan() {
-			agent := strings.ToLower(strings.TrimSpace(scanner.Text()))
-			userAgentBlockList = append(userAgentBlockList, agent)
+		agents, err := readUserAgents(resp.Body)
+		if err != nil {
+			return err
 		}
+		userAgentBlockList = append(userAgentBlockList, agents...)
 	}
 
 	b.userAgentBlockList = userAgentBlockList
@@ -122,49 +165,63 @@ func New(ctx context.Context, next http.Handler, config *Config, name string) (h
 		next:   next,
 		Config: *config,
 	}
-	err = blocker.Update()
+	err = blocker.update()
 	if err != nil {
 		return nil, fmt.Errorf("failed to update blocklists: %s", err)
 	}
 	return &blocker, nil
 }
 
 func (b *BotBlocker) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
-	if time.Now().Sub(b.lastUpdated) > time.Duration(time.Hour) {
-		err := b.Update()
+	if time.Since(b.lastUpdated) > time.Hour {
+		err := b.update()
 		if err != nil {
 			log.Errorf("failed to update blocklist: %v", err)
 		}
 	}
 	startTime := time.Now()
-	log.Debugf("Checking request: IP: \"%v\" user agent: \"%s\"", req.RemoteAddr, req.UserAgent())
+	log.Debugf("Checking request: CIDR: \"%v\" user agent: \"%s\"", req.RemoteAddr, req.UserAgent())
+	timer := func() {
+		log.Debugf("Checked request in %v", time.Since(startTime))
+	}
+	defer timer()
 
 	remoteAddrPort, err := netip.ParseAddrPort(req.RemoteAddr)
 	if err != nil {
 		http.Error(rw, "internal error", http.StatusInternalServerError)
 		return
 	}
-	remoteAddr := remoteAddrPort.Addr()
+	if b.shouldBlockIp(remoteAddrPort.Addr()) {
+		log.Infof("blocked request with from IP %v", remoteAddrPort.Addr())
+		http.Error(rw, "blocked", http.StatusForbidden)
+		return
+	}
+
+	agent := strings.ToLower(req.UserAgent())
+	if b.shouldBlockAgent(agent) {
+		log.Infof("blocked request with user agent %v because it contained %v", agent, agent)
+		http.Error(rw, "blocked", http.StatusForbidden)
+		return
+	}
+
+	b.next.ServeHTTP(rw, req)
+}
 
-	for _, badIP := range b.ipBlocklist {
-		if remoteAddr == badIP {
-			log.Infof("blocked request with from IP %v", remoteAddrPort.Addr())
-			log.Debugf("Checked request in %v", time.Now().Sub(startTime))
-			http.Error(rw, "blocked", http.StatusForbidden)
-			return
+func (b *BotBlocker) shouldBlockIp(addr netip.Addr) bool {
+	for _, badPrefix := range b.prefixBlocklist {
+		if badPrefix.Contains(addr) {
+			return true
 		}
 	}
+	return false
+}
 
-	agent := strings.ToLower(req.UserAgent())
+func (b *BotBlocker) shouldBlockAgent(userAgent string) bool {
+	userAgent = strings.ToLower(strings.TrimSpace(userAgent))
 	for _, badAgent := range b.userAgentBlockList {
-		if strings.Contains(agent, badAgent) {
-			log.Infof("blocked request with user agent %v because it contained %v", agent, badAgent)
-			log.Debugf("Checked request in %v", time.Now().Sub(startTime))
-			http.Error(rw, "blocked", http.StatusForbidden)
-			return
+		if strings.Contains(userAgent, badAgent) {
+			return true
 		}
 	}
-
-	log.Debugf("Checked request in %v", time.Now().Sub(startTime))
-	b.next.ServeHTTP(rw, req)
+	return false
 }