From 0a6e68f18413f41804b6c01ddc0d20721a5cce0a Mon Sep 17 00:00:00 2001 From: soulteary Date: Mon, 13 Jun 2022 12:07:20 +0800 Subject: [PATCH] chore: refactor choose mirrors --- README.md | 8 ++++---- cli/daemon.go | 2 +- linux/benchmark.go | 8 ++++---- linux/benchmark_test.go | 9 +++------ linux/common.go | 29 +++++++++++++++++++++++++--- linux/mirrors.go | 42 +++++++++++++++++++++-------------------- linux/mirrors_test.go | 30 ++++++++++++++++++++++------- linux/rewriter.go | 15 ++++++--------- proxy/proxy.go | 14 +++++--------- 9 files changed, 94 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 0dd8cd0..f7944d3 100644 --- a/README.md +++ b/README.md @@ -106,12 +106,12 @@ go run apt-proxy.go # go test -cover ./... ? github.com/soulteary/apt-proxy [no test files] -ok github.com/soulteary/apt-proxy/cli 2.492s coverage: 68.4% of statements -ok github.com/soulteary/apt-proxy/linux 7.420s coverage: 76.7% of statements -ok github.com/soulteary/apt-proxy/pkgs/httpcache 2.575s coverage: 82.7% of statements +ok github.com/soulteary/apt-proxy/cli 1.433s coverage: 69.2% of statements +ok github.com/soulteary/apt-proxy/linux 8.099s coverage: 80.2% of statements +ok github.com/soulteary/apt-proxy/pkgs/httpcache 2.357s coverage: 82.7% of statements ? github.com/soulteary/apt-proxy/pkgs/httplog [no test files] ok github.com/soulteary/apt-proxy/pkgs/stream.v1 1.238s coverage: 100.0% of statements -ok github.com/soulteary/apt-proxy/pkgs/vfs 2.255s coverage: 59.4% of statements +ok github.com/soulteary/apt-proxy/pkgs/vfs 0.930s coverage: 59.3% of statements ? github.com/soulteary/apt-proxy/proxy [no test files] ``` diff --git a/cli/daemon.go b/cli/daemon.go index 94cdbe0..f2cb825 100644 --- a/cli/daemon.go +++ b/cli/daemon.go @@ -27,7 +27,7 @@ func initStore(appFlags AppFlags) (cache httpcache.Cache, err error) { } func initProxy(appFlags AppFlags, cache httpcache.Cache) (ap *proxy.AptProxy) { - ap = proxy.NewAptProxyFromDefaults(appFlags.Mirror, appFlags.Mode) + ap = proxy.CreateAptProxyRouter(appFlags.Mirror, appFlags.Mode) ap.Handler = httpcache.NewHandler(cache, ap.Handler) return ap } diff --git a/linux/benchmark.go b/linux/benchmark.go index 3d51861..8f26743 100644 --- a/linux/benchmark.go +++ b/linux/benchmark.go @@ -40,11 +40,11 @@ type benchmarkResult struct { Duration time.Duration } -func getTheFastestMirror(m Mirrors, testUrl string) (string, error) { +func getTheFastestMirror(mirrors []string, testUrl string) (string, error) { ch := make(chan benchmarkResult) log.Printf("Start benchmarking mirrors") // kick off all benchmarks in parallel - for _, url := range m.URLs { + for _, url := range mirrors { go func(u string) { duration, err := benchmark(u, testUrl, BENCHMARK_MAX_TRIES) if err == nil { @@ -53,8 +53,8 @@ func getTheFastestMirror(m Mirrors, testUrl string) (string, error) { }(url) } - maxTries := len(m.URLs) - if 3 < maxTries { + maxTries := len(mirrors) + if maxTries > 3 { maxTries = 3 } diff --git a/linux/benchmark_test.go b/linux/benchmark_test.go index 8b446b9..747eef4 100644 --- a/linux/benchmark_test.go +++ b/linux/benchmark_test.go @@ -6,18 +6,15 @@ import ( ) func TestResourceBenchmark(t *testing.T) { - _, err := benchmark(UBUNTU_MIRROR_URLS, "", BENCHMARK_MAX_TRIES) + const resourcePath = "" + _, err := benchmark(UBUNTU_GEO_MIRROR_API, resourcePath, BENCHMARK_MAX_TRIES) if err != nil { t.Fatal(err) } } func TestMirrorsBenchmark(t *testing.T) { - mirrors, err := getGeoMirrors(UBUNTU_MIRROR_URLS) - if err != nil { - t.Fatal(err) - } - + mirrors := getGeoMirrorUrlsByMode(LINUX_DISTROS_UBUNTU) mirror, err := getTheFastestMirror(mirrors, UBUNTU_BENCHMAKR_URL) if err != nil { t.Fatal(err) diff --git a/linux/common.go b/linux/common.go index c425c23..9483a7f 100644 --- a/linux/common.go +++ b/linux/common.go @@ -25,7 +25,7 @@ const ( DEBIAN_BENCHMAKR_URL = "dists/bullseye/main/binary-amd64/Release" ) -var DEBIAN_MIRROR_URLS = []string{ +var BUILDIN_DEBIAN_MIRRORS = []string{ "http://ftp.cn.debian.org/debian/", "http://mirror.bjtu.edu.cn/debian/", "http://mirror.lzu.edu.cn/debian/", @@ -59,10 +59,33 @@ var DEBIAN_DEFAULT_CACHE_RULES = []Rule{ // Ubuntu const ( - UBUNTU_MIRROR_URLS = "http://mirrors.ubuntu.com/mirrors.txt" - UBUNTU_BENCHMAKR_URL = "dists/jammy/main/binary-amd64/Release" + UBUNTU_GEO_MIRROR_API = "http://mirrors.ubuntu.com/mirrors.txt" + UBUNTU_BENCHMAKR_URL = "dists/jammy/main/binary-amd64/Release" ) +var BUILDIN_UBUNTU_MIRRORS = []string{ + "http://mirrors.aliyun.com/ubuntu/", + "http://mirrors.huaweicloud.com/repository/ubuntu/", + "http://mirror.dlut.edu.cn/ubuntu/", + "http://mirrors.dgut.edu.cn/ubuntu/", + "http://mirrors.njupt.edu.cn/ubuntu/", + "https://mirrors.hit.edu.cn/ubuntu/", + "http://mirrors.yun-idc.com/ubuntu/", + "http://ftp.sjtu.edu.cn/ubuntu/", + "https://mirror.nju.edu.cn/ubuntu/", + "https://mirrors.bupt.edu.cn/ubuntu/", + "http://mirrors.skyshe.cn/ubuntu/", + "https://repo.huaweicloud.com/ubuntu/", + "http://mirror.lzu.edu.cn/ubuntu/", + "http://mirrors.cn99.com/ubuntu/", + "http://mirrors.cqu.edu.cn/ubuntu/", + "https://mirrors.cloud.tencent.com/ubuntu/", + "https://mirrors.ustc.edu.cn/ubuntu/", + "https://mirror.bjtu.edu.cn/ubuntu/", + "http://mirrors.sohu.com/ubuntu/", + "http://archive.ubuntu.com/ubuntu/", +} + var UBUNTU_HOST_PATTERN = regexp.MustCompile( `https?://(security|archive).ubuntu.com/ubuntu/(.+)$`, ) diff --git a/linux/mirrors.go b/linux/mirrors.go index d199b21..f95155a 100644 --- a/linux/mirrors.go +++ b/linux/mirrors.go @@ -3,45 +3,47 @@ package linux import ( "bufio" "net/http" - "net/url" "regexp" ) -type Mirrors struct { - URLs []string -} - -func getGeoMirrors(mirrorListUrl string) (m Mirrors, err error) { - if len(mirrorListUrl) == 0 { - m.URLs = DEBIAN_MIRROR_URLS - return m, nil +func getGeoMirrorUrlsByMode(mode string) (mirrors []string) { + if mode == LINUX_DISTROS_UBUNTU { + ubuntuMirrorsOnline, err := getUbuntuMirrorUrlsByGeo() + if err != nil { + return BUILDIN_UBUNTU_MIRRORS + } + return ubuntuMirrorsOnline } - _, err = url.ParseRequestURI(mirrorListUrl) - if err != nil { - return m, nil + if mode == LINUX_DISTROS_DEBIAN { + return BUILDIN_DEBIAN_MIRRORS } - response, err := http.Get(mirrorListUrl) + mirrors = append(mirrors, BUILDIN_UBUNTU_MIRRORS...) + mirrors = append(mirrors, BUILDIN_DEBIAN_MIRRORS...) + return mirrors +} + +func getUbuntuMirrorUrlsByGeo() (mirrors []string, err error) { + response, err := http.Get(UBUNTU_GEO_MIRROR_API) if err != nil { - return + return mirrors, err } defer response.Body.Close() scanner := bufio.NewScanner(response.Body) - m.URLs = []string{} for scanner.Scan() { - m.URLs = append(m.URLs, scanner.Text()) + mirrors = append(mirrors, scanner.Text()) } - return m, scanner.Err() + return mirrors, scanner.Err() } -func getPredefinedConfiguration(osType string) (string, string, *regexp.Regexp) { +func getPredefinedConfiguration(osType string) (string, *regexp.Regexp) { if osType == LINUX_DISTROS_UBUNTU { - return UBUNTU_MIRROR_URLS, UBUNTU_BENCHMAKR_URL, UBUNTU_HOST_PATTERN + return UBUNTU_BENCHMAKR_URL, UBUNTU_HOST_PATTERN } else { - return "", DEBIAN_BENCHMAKR_URL, DEBIAN_HOST_PATTERN + return DEBIAN_BENCHMAKR_URL, DEBIAN_HOST_PATTERN } } diff --git a/linux/mirrors_test.go b/linux/mirrors_test.go index faf7470..1b9eb6d 100644 --- a/linux/mirrors_test.go +++ b/linux/mirrors_test.go @@ -5,31 +5,47 @@ import ( ) func TestGetGeoMirrors(t *testing.T) { - mirrors, err := getGeoMirrors(UBUNTU_MIRROR_URLS) + mirrors, err := getUbuntuMirrorUrlsByGeo() if err != nil { t.Fatal(err) } - if len(mirrors.URLs) == 0 { + if len(mirrors) == 0 { + t.Fatal("No mirrors found") + } +} + +func TestGetMirrorUrlsByGeo(t *testing.T) { + mirrors := getGeoMirrorUrlsByMode(LINUX_ALL_DISTROS) + if len(mirrors) == 0 { + t.Fatal("No mirrors found") + } + + mirrors = getGeoMirrorUrlsByMode(LINUX_DISTROS_DEBIAN) + if len(mirrors) != len(BUILDIN_DEBIAN_MIRRORS) { + t.Fatal("Get mirrors error") + } + + mirrors = getGeoMirrorUrlsByMode(LINUX_DISTROS_UBUNTU) + if len(mirrors) == 0 { t.Fatal("No mirrors found") } } func TestGetPredefinedConfiguration(t *testing.T) { - url, res, pattern := getPredefinedConfiguration(LINUX_DISTROS_UBUNTU) - if url != UBUNTU_MIRROR_URLS || res != UBUNTU_BENCHMAKR_URL { + res, pattern := getPredefinedConfiguration(LINUX_DISTROS_UBUNTU) + if res != UBUNTU_BENCHMAKR_URL { t.Fatal("Failed to get resource link") } if !pattern.MatchString("http://archive.ubuntu.com/ubuntu/InRelease") { t.Fatal("Failed to verify domain name rules") } - url, res, pattern = getPredefinedConfiguration(LINUX_DISTROS_DEBIAN) - if url != "" || res != DEBIAN_BENCHMAKR_URL { + res, pattern = getPredefinedConfiguration(LINUX_DISTROS_DEBIAN) + if res != DEBIAN_BENCHMAKR_URL { t.Fatal("Failed to get resource link") } if !pattern.MatchString("http://deb.debian.org/debian/InRelease") { t.Fatal("Failed to verify domain name rules") } - } diff --git a/linux/rewriter.go b/linux/rewriter.go index 4bd3db4..e064612 100644 --- a/linux/rewriter.go +++ b/linux/rewriter.go @@ -16,26 +16,23 @@ type URLRewriter struct { func NewRewriter(mirrorUrl string, osType string) *URLRewriter { u := &URLRewriter{} + // user specify mirror + // TODO Both systems are supported if len(mirrorUrl) > 0 { mirror, err := url.Parse(mirrorUrl) if err == nil { log.Printf("using specify mirror %s", mirrorUrl) u.mirror = mirror - _, _, pattern := getPredefinedConfiguration(osType) + _, pattern := getPredefinedConfiguration(osType) u.pattern = pattern return u } } - mirrorsListUrl, benchmarkUrl, pattern := getPredefinedConfiguration(osType) + benchmarkUrl, pattern := getPredefinedConfiguration(osType) u.pattern = pattern - - mirrors, err := getGeoMirrors(mirrorsListUrl) - if err != nil { - log.Fatal(err) - } - - mirrorUrl, err = getTheFastestMirror(mirrors, benchmarkUrl) + mirrors := getGeoMirrorUrlsByMode(osType) + mirrorUrl, err := getTheFastestMirror(mirrors, benchmarkUrl) if err != nil { log.Println("Error finding fastest mirror", err) } diff --git a/proxy/proxy.go b/proxy/proxy.go index 5bb89c7..ef94ed6 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -22,7 +22,7 @@ type AptProxy struct { Rules []linux.Rule } -func NewAptProxyFromDefaults(mirror string, osType string) *AptProxy { +func CreateAptProxyRouter(mirror string, osType string) *AptProxy { rewriter = linux.NewRewriter(mirror, osType) var rules []linux.Rule if osType == linux.LINUX_DISTROS_UBUNTU { @@ -45,20 +45,16 @@ func (ap *AptProxy) ServeHTTP(rw http.ResponseWriter, r *http.Request) { if match { r.Header.Del("Cache-Control") if rule.Rewrite { - ap.rewriteRequest(r) + before := r.URL.String() + linux.Rewrite(r, rewriter) + log.Printf("rewrote %q to %q", before, r.URL.String()) + r.Host = r.URL.Host } } ap.Handler.ServeHTTP(&responseWriter{rw, rule}, r) } -func (ap *AptProxy) rewriteRequest(r *http.Request) { - before := r.URL.String() - linux.Rewrite(r, rewriter) - log.Printf("rewrote %q to %q", before, r.URL.String()) - r.Host = r.URL.Host -} - type responseWriter struct { http.ResponseWriter rule *linux.Rule