chore: refactor choose mirrors

README.md

@@ -106,12 +106,12 @@ go run apt-proxy.go
 # go test -cover ./...
 
 ?   	github.com/soulteary/apt-proxy	[no test files]
-ok  	github.com/soulteary/apt-proxy/cli	2.492s	coverage: 68.4% of statements
-ok  	github.com/soulteary/apt-proxy/linux	7.420s	coverage: 76.7% of statements
-ok  	github.com/soulteary/apt-proxy/pkgs/httpcache	2.575s	coverage: 82.7% of statements
+ok  	github.com/soulteary/apt-proxy/cli	1.433s	coverage: 69.2% of statements
+ok  	github.com/soulteary/apt-proxy/linux	8.099s	coverage: 80.2% of statements
+ok  	github.com/soulteary/apt-proxy/pkgs/httpcache	2.357s	coverage: 82.7% of statements
 ?   	github.com/soulteary/apt-proxy/pkgs/httplog	[no test files]
 ok  	github.com/soulteary/apt-proxy/pkgs/stream.v1	1.238s	coverage: 100.0% of statements
-ok  	github.com/soulteary/apt-proxy/pkgs/vfs	2.255s	coverage: 59.4% of statements
+ok  	github.com/soulteary/apt-proxy/pkgs/vfs	0.930s	coverage: 59.3% of statements
 ?   	github.com/soulteary/apt-proxy/proxy	[no test files]
 ```
 

cli/daemon.go

@@ -27,7 +27,7 @@ func initStore(appFlags AppFlags) (cache httpcache.Cache, err error) {
 }
 
 func initProxy(appFlags AppFlags, cache httpcache.Cache) (ap *proxy.AptProxy) {
-	ap = proxy.NewAptProxyFromDefaults(appFlags.Mirror, appFlags.Mode)
+	ap = proxy.CreateAptProxyRouter(appFlags.Mirror, appFlags.Mode)
 	ap.Handler = httpcache.NewHandler(cache, ap.Handler)
 	return ap
 }

linux/benchmark.go

@@ -40,11 +40,11 @@ type benchmarkResult struct {
 	Duration time.Duration
 }
 
-func getTheFastestMirror(m Mirrors, testUrl string) (string, error) {
+func getTheFastestMirror(mirrors []string, testUrl string) (string, error) {
 	ch := make(chan benchmarkResult)
 	log.Printf("Start benchmarking mirrors")
 	// kick off all benchmarks in parallel
-	for _, url := range m.URLs {
+	for _, url := range mirrors {
 		go func(u string) {
 			duration, err := benchmark(u, testUrl, BENCHMARK_MAX_TRIES)
 			if err == nil {
@@ -53,8 +53,8 @@ func getTheFastestMirror(m Mirrors, testUrl string) (string, error) {
 		}(url)
 	}
 
-	maxTries := len(m.URLs)
-	if 3 < maxTries {
+	maxTries := len(mirrors)
+	if maxTries > 3 {
 		maxTries = 3
 	}
 

linux/benchmark_test.go

@@ -6,18 +6,15 @@ import (
 )
 
 func TestResourceBenchmark(t *testing.T) {
-	_, err := benchmark(UBUNTU_MIRROR_URLS, "", BENCHMARK_MAX_TRIES)
+	const resourcePath = ""
+	_, err := benchmark(UBUNTU_GEO_MIRROR_API, resourcePath, BENCHMARK_MAX_TRIES)
 	if err != nil {
 		t.Fatal(err)
 	}
 }
 
 func TestMirrorsBenchmark(t *testing.T) {
-	mirrors, err := getGeoMirrors(UBUNTU_MIRROR_URLS)
-	if err != nil {
-		t.Fatal(err)
-	}
-
+	mirrors := getGeoMirrorUrlsByMode(LINUX_DISTROS_UBUNTU)
 	mirror, err := getTheFastestMirror(mirrors, UBUNTU_BENCHMAKR_URL)
 	if err != nil {
 		t.Fatal(err)

linux/common.go

@@ -25,7 +25,7 @@ const (
 	DEBIAN_BENCHMAKR_URL = "dists/bullseye/main/binary-amd64/Release"
 )
 
-var DEBIAN_MIRROR_URLS = []string{
+var BUILDIN_DEBIAN_MIRRORS = []string{
 	"http://ftp.cn.debian.org/debian/",
 	"http://mirror.bjtu.edu.cn/debian/",
 	"http://mirror.lzu.edu.cn/debian/",
@@ -59,10 +59,33 @@ var DEBIAN_DEFAULT_CACHE_RULES = []Rule{
 
 // Ubuntu
 const (
-	UBUNTU_MIRROR_URLS   = "http://mirrors.ubuntu.com/mirrors.txt"
-	UBUNTU_BENCHMAKR_URL = "dists/jammy/main/binary-amd64/Release"
+	UBUNTU_GEO_MIRROR_API = "http://mirrors.ubuntu.com/mirrors.txt"
+	UBUNTU_BENCHMAKR_URL  = "dists/jammy/main/binary-amd64/Release"
 )
 
+var BUILDIN_UBUNTU_MIRRORS = []string{
+	"http://mirrors.aliyun.com/ubuntu/",
+	"http://mirrors.huaweicloud.com/repository/ubuntu/",
+	"http://mirror.dlut.edu.cn/ubuntu/",
+	"http://mirrors.dgut.edu.cn/ubuntu/",
+	"http://mirrors.njupt.edu.cn/ubuntu/",
+	"https://mirrors.hit.edu.cn/ubuntu/",
+	"http://mirrors.yun-idc.com/ubuntu/",
+	"http://ftp.sjtu.edu.cn/ubuntu/",
+	"https://mirror.nju.edu.cn/ubuntu/",
+	"https://mirrors.bupt.edu.cn/ubuntu/",
+	"http://mirrors.skyshe.cn/ubuntu/",
+	"https://repo.huaweicloud.com/ubuntu/",
+	"http://mirror.lzu.edu.cn/ubuntu/",
+	"http://mirrors.cn99.com/ubuntu/",
+	"http://mirrors.cqu.edu.cn/ubuntu/",
+	"https://mirrors.cloud.tencent.com/ubuntu/",
+	"https://mirrors.ustc.edu.cn/ubuntu/",
+	"https://mirror.bjtu.edu.cn/ubuntu/",
+	"http://mirrors.sohu.com/ubuntu/",
+	"http://archive.ubuntu.com/ubuntu/",
+}
+
 var UBUNTU_HOST_PATTERN = regexp.MustCompile(
 	`https?://(security|archive).ubuntu.com/ubuntu/(.+)$`,
 )

linux/mirrors.go

@@ -3,45 +3,47 @@ package linux
 import (
 	"bufio"
 	"net/http"
-	"net/url"
 	"regexp"
 )
 
-type Mirrors struct {
-	URLs []string
-}
-
-func getGeoMirrors(mirrorListUrl string) (m Mirrors, err error) {
-	if len(mirrorListUrl) == 0 {
-		m.URLs = DEBIAN_MIRROR_URLS
-		return m, nil
+func getGeoMirrorUrlsByMode(mode string) (mirrors []string) {
+	if mode == LINUX_DISTROS_UBUNTU {
+		ubuntuMirrorsOnline, err := getUbuntuMirrorUrlsByGeo()
+		if err != nil {
+			return BUILDIN_UBUNTU_MIRRORS
+		}
+		return ubuntuMirrorsOnline
 	}
 
-	_, err = url.ParseRequestURI(mirrorListUrl)
-	if err != nil {
-		return m, nil
+	if mode == LINUX_DISTROS_DEBIAN {
+		return BUILDIN_DEBIAN_MIRRORS
 	}
 
-	response, err := http.Get(mirrorListUrl)
+	mirrors = append(mirrors, BUILDIN_UBUNTU_MIRRORS...)
+	mirrors = append(mirrors, BUILDIN_DEBIAN_MIRRORS...)
+	return mirrors
+}
+
+func getUbuntuMirrorUrlsByGeo() (mirrors []string, err error) {
+	response, err := http.Get(UBUNTU_GEO_MIRROR_API)
 	if err != nil {
-		return
+		return mirrors, err
 	}
 
 	defer response.Body.Close()
 	scanner := bufio.NewScanner(response.Body)
-	m.URLs = []string{}
 
 	for scanner.Scan() {
-		m.URLs = append(m.URLs, scanner.Text())
+		mirrors = append(mirrors, scanner.Text())
 	}
 
-	return m, scanner.Err()
+	return mirrors, scanner.Err()
 }
 
-func getPredefinedConfiguration(osType string) (string, string, *regexp.Regexp) {
+func getPredefinedConfiguration(osType string) (string, *regexp.Regexp) {
 	if osType == LINUX_DISTROS_UBUNTU {
-		return UBUNTU_MIRROR_URLS, UBUNTU_BENCHMAKR_URL, UBUNTU_HOST_PATTERN
+		return UBUNTU_BENCHMAKR_URL, UBUNTU_HOST_PATTERN
 	} else {
-		return "", DEBIAN_BENCHMAKR_URL, DEBIAN_HOST_PATTERN
+		return DEBIAN_BENCHMAKR_URL, DEBIAN_HOST_PATTERN
 	}
 }

linux/mirrors_test.go

@@ -5,31 +5,47 @@ import (
 )
 
 func TestGetGeoMirrors(t *testing.T) {
-	mirrors, err := getGeoMirrors(UBUNTU_MIRROR_URLS)
+	mirrors, err := getUbuntuMirrorUrlsByGeo()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if len(mirrors.URLs) == 0 {
+	if len(mirrors) == 0 {
+		t.Fatal("No mirrors found")
+	}
+}
+
+func TestGetMirrorUrlsByGeo(t *testing.T) {
+	mirrors := getGeoMirrorUrlsByMode(LINUX_ALL_DISTROS)
+	if len(mirrors) == 0 {
+		t.Fatal("No mirrors found")
+	}
+
+	mirrors = getGeoMirrorUrlsByMode(LINUX_DISTROS_DEBIAN)
+	if len(mirrors) != len(BUILDIN_DEBIAN_MIRRORS) {
+		t.Fatal("Get mirrors error")
+	}
+
+	mirrors = getGeoMirrorUrlsByMode(LINUX_DISTROS_UBUNTU)
+	if len(mirrors) == 0 {
 		t.Fatal("No mirrors found")
 	}
 }
 
 func TestGetPredefinedConfiguration(t *testing.T) {
-	url, res, pattern := getPredefinedConfiguration(LINUX_DISTROS_UBUNTU)
-	if url != UBUNTU_MIRROR_URLS || res != UBUNTU_BENCHMAKR_URL {
+	res, pattern := getPredefinedConfiguration(LINUX_DISTROS_UBUNTU)
+	if res != UBUNTU_BENCHMAKR_URL {
 		t.Fatal("Failed to get resource link")
 	}
 	if !pattern.MatchString("http://archive.ubuntu.com/ubuntu/InRelease") {
 		t.Fatal("Failed to verify domain name rules")
 	}
 
-	url, res, pattern = getPredefinedConfiguration(LINUX_DISTROS_DEBIAN)
-	if url != "" || res != DEBIAN_BENCHMAKR_URL {
+	res, pattern = getPredefinedConfiguration(LINUX_DISTROS_DEBIAN)
+	if res != DEBIAN_BENCHMAKR_URL {
 		t.Fatal("Failed to get resource link")
 	}
 	if !pattern.MatchString("http://deb.debian.org/debian/InRelease") {
 		t.Fatal("Failed to verify domain name rules")
 	}
-
 }

linux/rewriter.go

@@ -16,26 +16,23 @@ type URLRewriter struct {
 func NewRewriter(mirrorUrl string, osType string) *URLRewriter {
 	u := &URLRewriter{}
 
+	// user specify mirror
+	// TODO Both systems are supported
 	if len(mirrorUrl) > 0 {
 		mirror, err := url.Parse(mirrorUrl)
 		if err == nil {
 			log.Printf("using specify mirror %s", mirrorUrl)
 			u.mirror = mirror
-			_, _, pattern := getPredefinedConfiguration(osType)
+			_, pattern := getPredefinedConfiguration(osType)
 			u.pattern = pattern
 			return u
 		}
 	}
 
-	mirrorsListUrl, benchmarkUrl, pattern := getPredefinedConfiguration(osType)
+	benchmarkUrl, pattern := getPredefinedConfiguration(osType)
 	u.pattern = pattern
-
-	mirrors, err := getGeoMirrors(mirrorsListUrl)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	mirrorUrl, err = getTheFastestMirror(mirrors, benchmarkUrl)
+	mirrors := getGeoMirrorUrlsByMode(osType)
+	mirrorUrl, err := getTheFastestMirror(mirrors, benchmarkUrl)
 	if err != nil {
 		log.Println("Error finding fastest mirror", err)
 	}

proxy/proxy.go

@@ -22,7 +22,7 @@ type AptProxy struct {
 	Rules   []linux.Rule
 }
 
-func NewAptProxyFromDefaults(mirror string, osType string) *AptProxy {
+func CreateAptProxyRouter(mirror string, osType string) *AptProxy {
 	rewriter = linux.NewRewriter(mirror, osType)
 	var rules []linux.Rule
 	if osType == linux.LINUX_DISTROS_UBUNTU {
@@ -45,20 +45,16 @@ func (ap *AptProxy) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	if match {
 		r.Header.Del("Cache-Control")
 		if rule.Rewrite {
-			ap.rewriteRequest(r)
+			before := r.URL.String()
+			linux.Rewrite(r, rewriter)
+			log.Printf("rewrote %q to %q", before, r.URL.String())
+			r.Host = r.URL.Host
 		}
 	}
 
 	ap.Handler.ServeHTTP(&responseWriter{rw, rule}, r)
 }
 
-func (ap *AptProxy) rewriteRequest(r *http.Request) {
-	before := r.URL.String()
-	linux.Rewrite(r, rewriter)
-	log.Printf("rewrote %q to %q", before, r.URL.String())
-	r.Host = r.URL.Host
-}
-
 type responseWriter struct {
 	http.ResponseWriter
 	rule *linux.Rule