chore: refactor mirror manage, rewriter

cli/cli.go

@@ -3,7 +3,7 @@ package cli
 import (
 	"flag"
 
-	"github.com/soulteary/apt-proxy/linux"
+	Mirrors "github.com/soulteary/apt-proxy/internal/mirrors"
 	"github.com/soulteary/apt-proxy/state"
 )
 
@@ -13,22 +13,22 @@ const (
 	DEFAULT_CACHE_DIR     = "./.aptcache"
 	DEFAULT_UBUNTU_MIRROR = "" // "https://mirrors.tuna.tsinghua.edu.cn/ubuntu/"
 	DEFAULT_DEBIAN_MIRROR = "" // "https://mirrors.tuna.tsinghua.edu.cn/debian/"
-	DEFAULT_MODE_NAME     = linux.LINUX_ALL_DISTROS
+	DEFAULT_MODE_NAME     = Mirrors.LINUX_ALL_DISTROS
 	DEFAULT_DEBUG         = false
 )
 
 var Version string
 
 func getProxyMode(mode string) int {
-	if mode == linux.LINUX_DISTROS_UBUNTU {
-		return linux.TYPE_LINUX_DISTROS_UBUNTU
+	if mode == Mirrors.LINUX_DISTROS_UBUNTU {
+		return Mirrors.TYPE_LINUX_DISTROS_UBUNTU
 	}
 
-	if mode == linux.LINUX_DISTROS_DEBIAN {
-		return linux.TYPE_LINUX_DISTROS_DEBIAN
+	if mode == Mirrors.LINUX_DISTROS_DEBIAN {
+		return Mirrors.TYPE_LINUX_DISTROS_DEBIAN
 	}
 
-	return linux.TYPE_LINUX_ALL_DISTROS
+	return Mirrors.TYPE_LINUX_ALL_DISTROS
 }
 
 func ParseFlags() (appFlags AppFlags) {

cli/cli_test.go

@@ -4,19 +4,19 @@ import (
 	"os"
 	"testing"
 
-	"github.com/soulteary/apt-proxy/linux"
+	Mirrors "github.com/soulteary/apt-proxy/internal/mirrors"
 )
 
 func TestGetProxyMode(t *testing.T) {
-	if getProxyMode("not-support-os") != linux.TYPE_LINUX_ALL_DISTROS {
+	if getProxyMode("not-support-os") != Mirrors.TYPE_LINUX_ALL_DISTROS {
 		t.Fatal("Incorrect return default value")
 	}
 
-	if getProxyMode(linux.LINUX_DISTROS_DEBIAN) != linux.TYPE_LINUX_DISTROS_DEBIAN {
+	if getProxyMode(Mirrors.LINUX_DISTROS_DEBIAN) != Mirrors.TYPE_LINUX_DISTROS_DEBIAN {
 		t.Fatal("Incorrect return value")
 	}
 
-	if getProxyMode(linux.LINUX_DISTROS_UBUNTU) != linux.TYPE_LINUX_DISTROS_UBUNTU {
+	if getProxyMode(Mirrors.LINUX_DISTROS_UBUNTU) != Mirrors.TYPE_LINUX_DISTROS_UBUNTU {
 		t.Fatal("Incorrect return value")
 	}
 }

linux/benchmark.go → internal/mirrors/benchmark.go

@@ -1,4 +1,4 @@
-package linux
+package mirrors
 
 import (
 	"errors"
@@ -40,7 +40,7 @@ type benchmarkResult struct {
 	Duration time.Duration
 }
 
-func getTheFastestMirror(mirrors []string, testUrl string) (string, error) {
+func GetTheFastestMirror(mirrors []string, testUrl string) (string, error) {
 	ch := make(chan benchmarkResult)
 	log.Printf("Start benchmarking mirrors")
 	// kick off all benchmarks in parallel

linux/benchmark_test.go → internal/mirrors/benchmark_test.go

@@ -1,4 +1,4 @@
-package linux
+package mirrors
 
 import (
 	"log"
@@ -14,8 +14,8 @@ func TestResourceBenchmark(t *testing.T) {
 }
 
 func TestMirrorsBenchmark(t *testing.T) {
-	mirrors := getGeoMirrorUrlsByMode(TYPE_LINUX_DISTROS_UBUNTU)
-	mirror, err := getTheFastestMirror(mirrors, UBUNTU_BENCHMAKR_URL)
+	mirrors := GetGeoMirrorUrlsByMode(TYPE_LINUX_DISTROS_UBUNTU)
+	mirror, err := GetTheFastestMirror(mirrors, UBUNTU_BENCHMAKR_URL)
 	if err != nil {
 		t.Fatal(err)
 	}

linux/common.go → internal/mirrors/common.go

@@ -1,6 +1,9 @@
-package linux
+package mirrors
 
-import "regexp"
+import (
+	"fmt"
+	"regexp"
+)
 
 const (
 	LINUX_ALL_DISTROS    string = "all"
@@ -110,3 +113,8 @@ var UBUNTU_DEFAULT_CACHE_RULES = []Rule{
 	// Add file file hash
 	{Pattern: regexp.MustCompile(`/by-hash/`), CacheControl: `max-age=3600`, Rewrite: true, OS: TYPE_LINUX_DISTROS_UBUNTU},
 }
+
+func (r *Rule) String() string {
+	return fmt.Sprintf("%s Cache-Control=%s Rewrite=%#v",
+		r.Pattern.String(), r.CacheControl, r.Rewrite)
+}

internal/mirrors/mirrors.go

@@ -1,5 +1,11 @@
 package mirrors
 
+import (
+	"bufio"
+	"net/http"
+	"regexp"
+)
+
 type buildin_custom_mirror struct {
 	url   string
 	alias string
@@ -34,3 +40,45 @@ func GetCentOSMirrorByAliases(alias string) string {
 	}
 	return ""
 }
+
+func GetGeoMirrorUrlsByMode(mode int) (mirrors []string) {
+	if mode == TYPE_LINUX_DISTROS_UBUNTU {
+		ubuntuMirrorsOnline, err := getUbuntuMirrorUrlsByGeo()
+		if err != nil {
+			return BUILDIN_OFFICAL_UBUNTU_MIRRORS
+		}
+		return ubuntuMirrorsOnline
+	}
+
+	if mode == TYPE_LINUX_DISTROS_DEBIAN {
+		return BUILDIN_OFFICAL_DEBIAN_MIRRORS
+	}
+
+	mirrors = append(mirrors, BUILDIN_OFFICAL_UBUNTU_MIRRORS...)
+	mirrors = append(mirrors, BUILDIN_OFFICAL_DEBIAN_MIRRORS...)
+	return mirrors
+}
+
+func getUbuntuMirrorUrlsByGeo() (mirrors []string, err error) {
+	response, err := http.Get(UBUNTU_GEO_MIRROR_API)
+	if err != nil {
+		return mirrors, err
+	}
+
+	defer response.Body.Close()
+	scanner := bufio.NewScanner(response.Body)
+
+	for scanner.Scan() {
+		mirrors = append(mirrors, scanner.Text())
+	}
+
+	return mirrors, scanner.Err()
+}
+
+func GetPredefinedConfiguration(proxyMode int) (string, *regexp.Regexp) {
+	if proxyMode == TYPE_LINUX_DISTROS_UBUNTU {
+		return UBUNTU_BENCHMAKR_URL, UBUNTU_HOST_PATTERN
+	} else {
+		return DEBIAN_BENCHMAKR_URL, DEBIAN_HOST_PATTERN
+	}
+}

linux/mirrors_test.go → internal/mirrors/mirrors_test.go

@@ -1,4 +1,4 @@
-package linux
+package mirrors
 
 import (
 	"testing"
@@ -16,24 +16,24 @@ func TestGetGeoMirrors(t *testing.T) {
 }
 
 func TestGetMirrorUrlsByGeo(t *testing.T) {
-	mirrors := getGeoMirrorUrlsByMode(TYPE_LINUX_ALL_DISTROS)
+	mirrors := GetGeoMirrorUrlsByMode(TYPE_LINUX_ALL_DISTROS)
 	if len(mirrors) == 0 {
 		t.Fatal("No mirrors found")
 	}
 
-	mirrors = getGeoMirrorUrlsByMode(TYPE_LINUX_DISTROS_DEBIAN)
+	mirrors = GetGeoMirrorUrlsByMode(TYPE_LINUX_DISTROS_DEBIAN)
 	if len(mirrors) != len(BUILDIN_OFFICAL_DEBIAN_MIRRORS) {
 		t.Fatal("Get mirrors error")
 	}
 
-	mirrors = getGeoMirrorUrlsByMode(TYPE_LINUX_DISTROS_UBUNTU)
+	mirrors = GetGeoMirrorUrlsByMode(TYPE_LINUX_DISTROS_UBUNTU)
 	if len(mirrors) == 0 {
 		t.Fatal("No mirrors found")
 	}
 }
 
 func TestGetPredefinedConfiguration(t *testing.T) {
-	res, pattern := getPredefinedConfiguration(TYPE_LINUX_DISTROS_UBUNTU)
+	res, pattern := GetPredefinedConfiguration(TYPE_LINUX_DISTROS_UBUNTU)
 	if res != UBUNTU_BENCHMAKR_URL {
 		t.Fatal("Failed to get resource link")
 	}
@@ -47,7 +47,7 @@ func TestGetPredefinedConfiguration(t *testing.T) {
 		t.Fatal("Failed to verify domain name rules")
 	}
 
-	res, pattern = getPredefinedConfiguration(TYPE_LINUX_DISTROS_DEBIAN)
+	res, pattern = GetPredefinedConfiguration(TYPE_LINUX_DISTROS_DEBIAN)
 	if res != DEBIAN_BENCHMAKR_URL {
 		t.Fatal("Failed to get resource link")
 	}

linux/rewriter.go → internal/rewriter/rewriter.go

@@ -1,12 +1,12 @@
-package linux
+package rewriter
 
 import (
-	"fmt"
 	"log"
 	"net/http"
 	"net/url"
 	"regexp"
 
+	Mirrors "github.com/soulteary/apt-proxy/internal/mirrors"
 	"github.com/soulteary/apt-proxy/state"
 )
 
@@ -20,23 +20,23 @@ type URLRewriter struct {
 	pattern *regexp.Regexp
 }
 
-func GetRewriteRulesByMode(mode int) (rules []Rule) {
-	if mode == TYPE_LINUX_DISTROS_UBUNTU {
-		return UBUNTU_DEFAULT_CACHE_RULES
+func GetRewriteRulesByMode(mode int) (rules []Mirrors.Rule) {
+	if mode == Mirrors.TYPE_LINUX_DISTROS_UBUNTU {
+		return Mirrors.UBUNTU_DEFAULT_CACHE_RULES
 	}
-	if mode == TYPE_LINUX_DISTROS_DEBIAN {
-		return DEBIAN_DEFAULT_CACHE_RULES
+	if mode == Mirrors.TYPE_LINUX_DISTROS_DEBIAN {
+		return Mirrors.DEBIAN_DEFAULT_CACHE_RULES
 	}
 
-	rules = append(rules, UBUNTU_DEFAULT_CACHE_RULES...)
-	rules = append(rules, DEBIAN_DEFAULT_CACHE_RULES...)
+	rules = append(rules, Mirrors.UBUNTU_DEFAULT_CACHE_RULES...)
+	rules = append(rules, Mirrors.DEBIAN_DEFAULT_CACHE_RULES...)
 	return rules
 }
 
 func getRewriterForDebian() *URLRewriter {
 	u := &URLRewriter{}
 	debianMirror := state.GetDebianMirror()
-	benchmarkUrl, pattern := getPredefinedConfiguration(TYPE_LINUX_DISTROS_DEBIAN)
+	benchmarkUrl, pattern := Mirrors.GetPredefinedConfiguration(Mirrors.TYPE_LINUX_DISTROS_DEBIAN)
 	u.pattern = pattern
 
 	if debianMirror != nil {
@@ -45,8 +45,8 @@ func getRewriterForDebian() *URLRewriter {
 		return u
 	}
 
-	mirrors := getGeoMirrorUrlsByMode(TYPE_LINUX_DISTROS_DEBIAN)
-	fastest, err := getTheFastestMirror(mirrors, benchmarkUrl)
+	mirrors := Mirrors.GetGeoMirrorUrlsByMode(Mirrors.TYPE_LINUX_DISTROS_DEBIAN)
+	fastest, err := Mirrors.GetTheFastestMirror(mirrors, benchmarkUrl)
 	if err != nil {
 		log.Println("Error finding fastest mirror", err)
 	}
@@ -62,7 +62,7 @@ func getRewriterForDebian() *URLRewriter {
 func getRewriterForUbuntu() *URLRewriter {
 	u := &URLRewriter{}
 	ubuntuMirror := state.GetUbuntuMirror()
-	benchmarkUrl, pattern := getPredefinedConfiguration(TYPE_LINUX_DISTROS_UBUNTU)
+	benchmarkUrl, pattern := Mirrors.GetPredefinedConfiguration(Mirrors.TYPE_LINUX_DISTROS_UBUNTU)
 	u.pattern = pattern
 
 	if ubuntuMirror != nil {
@@ -71,8 +71,8 @@ func getRewriterForUbuntu() *URLRewriter {
 		return u
 	}
 
-	mirrors := getGeoMirrorUrlsByMode(TYPE_LINUX_DISTROS_UBUNTU)
-	fastest, err := getTheFastestMirror(mirrors, benchmarkUrl)
+	mirrors := Mirrors.GetGeoMirrorUrlsByMode(Mirrors.TYPE_LINUX_DISTROS_UBUNTU)
+	fastest, err := Mirrors.GetTheFastestMirror(mirrors, benchmarkUrl)
 	if err != nil {
 		log.Println("Error finding fastest mirror", err)
 	}
@@ -88,12 +88,12 @@ func getRewriterForUbuntu() *URLRewriter {
 func CreateNewRewriters(mode int) *URLRewriters {
 	rewriters := &URLRewriters{}
 
-	if mode == TYPE_LINUX_DISTROS_DEBIAN {
+	if mode == Mirrors.TYPE_LINUX_DISTROS_DEBIAN {
 		rewriters.debian = getRewriterForDebian()
 		return rewriters
 	}
 
-	if mode == TYPE_LINUX_DISTROS_UBUNTU {
+	if mode == Mirrors.TYPE_LINUX_DISTROS_UBUNTU {
 		rewriters.ubuntu = getRewriterForUbuntu()
 		return rewriters
 	}
@@ -106,7 +106,7 @@ func CreateNewRewriters(mode int) *URLRewriters {
 func RewriteRequestByMode(r *http.Request, rewriters *URLRewriters, mode int) {
 	uri := r.URL.String()
 	var rewriter *URLRewriter
-	if mode == TYPE_LINUX_DISTROS_UBUNTU {
+	if mode == Mirrors.TYPE_LINUX_DISTROS_UBUNTU {
 		rewriter = rewriters.ubuntu
 	} else {
 		// mode == TYPE_LINUX_DISTROS_DEBIAN
@@ -126,16 +126,11 @@ func RewriteRequestByMode(r *http.Request, rewriters *URLRewriters, mode int) {
 	}
 }
 
-func MatchingRule(subject string, rules []Rule) (*Rule, bool) {
+func MatchingRule(subject string, rules []Mirrors.Rule) (*Mirrors.Rule, bool) {
 	for _, rule := range rules {
 		if rule.Pattern.MatchString(subject) {
 			return &rule, true
 		}
 	}
 	return nil, false
 }
-
-func (r *Rule) String() string {
-	return fmt.Sprintf("%s Cache-Control=%s Rewrite=%#v",
-		r.Pattern.String(), r.CacheControl, r.Rewrite)
-}