init

README.md

@@ -0,0 +1,46 @@
+## 介绍
+
+使用`github.com/projectdiscovery/cdncheck`来快速判断是否存在CDN
+
+## 安装编译
+
+### 自动
+
+```shell
+go install github.com/damit5/cdnCheck_go@latest
+```
+
+
+
+### 手动
+
+```shell
+git clone https://github.com/damit5/cdnCheck_go
+cd cdnCheck_go
+
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -trimpath -o release/cdnCheck_go_linux_amd64
+GOOS=darwin CGO_ENABLED=0 GOARCH=amd64 go build -ldflags="-s -w" -trimpath -o release/cdnCheck_darwin
+GOOS=windows CGO_ENABLED=0 GOARCH=amd64 go build -ldflags="-s -w" -trimpath -o release/cdnCheck.exe
+```
+
+## 用法
+
+```shell
+Usage of ./cdnCheck_darwin:
+  -nd string
+    	无CDN域名保存地址，不保存置空即可
+  -ni string
+    	无CDN IP保存地址，不保存置空即可
+  -t string
+    	需要扫描的文件
+  -thread int
+    	并发数 (default 20)
+
+./cdnCheck_darwin -t ../test/baidu.com.txt -thread 100 -nd ../test/nocdndomain.txt -ni ../test/nocdnip.txt
+```
+
+![image-20220718142155174](README.assets/image-20220718142155174.png)
+
+## 疑难杂症
+
+* 建议修改本地NS Server的地址为`8.8.8.8`或者其他大DNS，不然可能因为并发太大导致结果很多请求被拒绝

go.mod

@@ -0,0 +1,15 @@
+module github.com/damit5/cdnCheck_go
+
+go 1.18
+
+require (
+	github.com/EDDYCJY/gsema v0.0.0-20190120044130-7f6a61b75219
+	github.com/projectdiscovery/cdncheck v0.0.3
+)
+
+require (
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/yl2chen/cidranger v1.0.2 // indirect
+)

go.sum

@@ -0,0 +1,34 @@
+github.com/EDDYCJY/gsema v0.0.0-20190120044130-7f6a61b75219 h1:TCY1aa2IQInhAIiumrIkZfUmk2+q0OjspCQutj+Kjng=
+github.com/EDDYCJY/gsema v0.0.0-20190120044130-7f6a61b75219/go.mod h1:wd3egVZ3+ISuA+PWDRLbjNRYZUQwwWXCTcXQrfZAOmQ=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/projectdiscovery/cdncheck v0.0.3 h1:li2/rUJmhVXSqRFyhJMqi6pdBX6ZxMnwzBfE0Kifj/g=
+github.com/projectdiscovery/cdncheck v0.0.3/go.mod h1:EevMeCG1ogBoUJYaa0Mv9R1VUboDm/DiynId7DboKy0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
+github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -0,0 +1,128 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"github.com/EDDYCJY/gsema"
+	"github.com/projectdiscovery/cdncheck"
+	"log"
+	"net"
+	"os"
+	"strings"
+)
+
+var client *cdncheck.Client
+var err error
+var semaphore *gsema.Semaphore
+var cdnResult []string	// 保存CDN的域名
+var nonCdnDomainResult []string // 保存无CDN的域名
+var nonCdnIPResult []string	// 保存无CDN的IP
+
+// 传入的参数
+var target string
+var nonCdnDomainSavePath string
+var nonCdnIPSavePath string
+var threads int
+
+func checkCDN(domain string) {
+	defer semaphore.Done()
+	// 域名解析成IP，可能有多个IP
+	ips, err := net.LookupIP(domain)
+	if err != nil {
+		log.Println(domain, err)
+		return
+	}
+
+	// 进行CDN验证
+	for _, ip := range ips {
+		// 过滤ipv6
+		if strings.Contains(ip.String(), ":") {
+			continue
+		}
+
+		if found, provider, err := client.Check(ip); found && err == nil {
+			log.Println(fmt.Sprintf("%s ==> %s is part of %s cdn", domain, ip.String(), provider))
+
+			cdnResult = append(cdnResult, domain)
+		} else {
+			log.Println(fmt.Sprintf("%s ==> %s has no cdn", domain, ip.String()))
+
+			nonCdnDomainResult = append(nonCdnDomainResult, domain)
+			nonCdnIPResult = append(nonCdnIPResult, ip.String())
+		}
+	}
+}
+
+// 数组去重
+func removeRepeatedElement(arr []string) (newArr []string) {
+	newArr = make([]string, 0)
+	for i := 0; i < len(arr); i++ {
+		repeat := false
+		for j := i + 1; j < len(arr); j++ {
+			if arr[i] == arr[j] {
+				repeat = true
+				break
+			}
+		}
+		if !repeat {
+			newArr = append(newArr, arr[i])
+		}
+	}
+	return
+}
+
+func flagInit() {
+	flag.StringVar(&target, "t", "", "需要扫描的文件")
+	flag.StringVar(&nonCdnDomainSavePath, "nd", "", "无CDN域名保存地址，不保存置空即可")
+	flag.StringVar(&nonCdnIPSavePath, "ni", "", "无CDN IP保存地址，不保存置空即可")
+	flag.IntVar(&threads, "thread", 20, "并发数")
+	flag.Parse()
+}
+
+func main() {
+	flagInit()
+	if target == "" {
+		flag.Usage()
+		return
+	}
+	semaphore = gsema.NewSemaphore(threads)
+	client, err = cdncheck.NewWithCache()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	file, err := os.Open(target)
+	defer file.Close()
+	if err != nil {
+		log.Panicln(err)
+		return
+	}
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		semaphore.Add(1)
+		go checkCDN(scanner.Text())
+	}
+	semaphore.Wait()
+
+	if nonCdnDomainSavePath != "" {	// 写入无CDN的domain
+		openFile, _ := os.OpenFile(nonCdnDomainSavePath, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0666)
+		defer openFile.Close()
+		write := bufio.NewWriter(openFile)
+		for _, domain := range removeRepeatedElement(nonCdnDomainResult) {
+			write.WriteString(domain + "\n")
+		}
+		write.Flush()
+	}
+
+	if nonCdnIPSavePath != "" {	// 写入无CDN的domain
+		openFile, _ := os.OpenFile(nonCdnIPSavePath, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0666)
+		defer openFile.Close()
+		write := bufio.NewWriter(openFile)
+		for _, ip := range removeRepeatedElement(nonCdnIPResult) {
+			write.WriteString(ip + "\n")
+		}
+		write.Flush()
+	}
+}