domain convert

convert/classical.go

@@ -44,7 +44,7 @@ func convertClassical(s *os.File, t *os.File) error {
 			res = "keyword:" + args[1]
 		case "DOMAIN-REGEX":
 			if conf.Convert.EnableRegex {
-				res = "regex:" + args[1]
+				res = "regexp:" + args[1]
 			} else {
 				log.L().Sugar().Warnf("regex is disabled: %s", line)
 				continue

convert/convert.go

@@ -35,8 +35,11 @@ func Convert(t Task) error {
 		if err != nil {
 			return errors.Wrap(err, "convert classical failed")
 		}
-	// case conf.TypeDomain:
-	// 	err = convertDomain(source, target)
+	case conf.TypeDomain:
+		err = convertDomain(source, target)
+		if err != nil {
+			return errors.Wrap(err, "convert domain failed")
+		}
 	default:
 		return errors.New("unknown convert type")
 	}

convert/domain.go

@@ -1 +1,75 @@
 package convert
+
+import (
+	"bufio"
+	"os"
+	"strings"
+
+	"github.com/dn-11/provider2domainset/conf"
+	"github.com/dn-11/provider2domainset/log"
+	"github.com/pkg/errors"
+)
+
+func convertDomain(s *os.File, t *os.File) error {
+	scanner := bufio.NewScanner(s)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		line = strings.Split(line, "#")[0]
+		line = strings.TrimSpace(line)
+
+		if line == "payload:" {
+			continue
+		}
+		line = strings.TrimPrefix(line, `- '`)
+		line = strings.TrimSuffix(line, `'`)
+
+		if len(line) == 0 {
+			continue
+		}
+
+		res := ""
+
+		if strings.Contains(line, "*") || line[0] == '.' {
+			if !conf.Convert.EnableRegex {
+				log.L().Sugar().Warnf("regex is disabled: %s", line)
+				continue
+			}
+
+			if line[0] == '.' {
+				if !strings.Contains(line, "*") {
+					res = "regexp:^.+" + strings.ReplaceAll(line, ".", "\\.") + "$"
+				} else {
+					res = "regexp:^.+" + strings.ReplaceAll(strings.ReplaceAll(line, ".", "\\."), "*", "[^.]+") + "$"
+					log.L().Sugar().Warnf("complex rule, check carefully: %s -> %s", line, res)
+				}
+			} else {
+				if line[0] != '+' {
+					res = "regexp:^" + strings.ReplaceAll(strings.ReplaceAll(line, ".", "\\."), "*", "[^.]+") + "$"
+				} else {
+					res = "regexp:^.*" + strings.ReplaceAll(strings.ReplaceAll(line[2:], ".", "\\."), "*", "[^.]+") + "$"
+					log.L().Sugar().Warnf("complex rule, check carefully: %s -> %s", line, res)
+				}
+			}
+		} else {
+			if line[0] == '+' {
+				res = "domain:" + line[2:]
+			} else {
+				res = "full:" + line
+			}
+		}
+
+		log.L().Sugar().Debugf("convert line: %s -> %s", line, res)
+
+		if _, err := t.WriteString(res + "\n"); err != nil {
+			return errors.Wrap(err, "write target file failed")
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return errors.Wrap(err, "scan source file failed")
+	}
+
+	return nil
+}

convert/domain_test.go

@@ -1 +1,60 @@
 package convert_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/dn-11/provider2domainset/conf"
+	"github.com/dn-11/provider2domainset/convert"
+	"github.com/dn-11/provider2domainset/log"
+	"go.uber.org/zap/zapcore"
+)
+
+func TestConvertDomainTxt(t *testing.T) {
+	log.L().SetLogLevel(zapcore.DebugLevel)
+	conf.Convert.EnableRegex = true
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		t.Error(err)
+	}
+
+	task := convert.Task{
+		Source: filepath.Join(pwd, "test_case"),
+		Target: filepath.Join(pwd, "test_output"),
+		File: conf.File{
+			Name: "domain.txt",
+			Type: "domain",
+		},
+	}
+
+	err = convert.Convert(task)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestConvertDomainYaml(t *testing.T) {
+	log.L().SetLogLevel(zapcore.DebugLevel)
+	conf.Convert.EnableRegex = true
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		t.Error(err)
+	}
+
+	task := convert.Task{
+		Source: filepath.Join(pwd, "test_case"),
+		Target: filepath.Join(pwd, "test_output"),
+		File: conf.File{
+			Name: "domain.yaml",
+			Type: "domain",
+		},
+	}
+
+	err = convert.Convert(task)
+	if err != nil {
+		t.Error(err)
+	}
+}

convert/test_case/domain.txt

@@ -4,3 +4,7 @@
 git.*.com
 *.baidu.com
 .163.com
+*.*.google.com
+.abc.*.com
++.exp.*.baidu.com
+*

convert/test_case/domain.yaml

@@ -6,3 +6,7 @@ payload:
   - 'git.*.com'
   - '*.baidu.com'
   - '.163.com'
+  - '*.*.google.com'
+  - '.abc.*.com'
+  - '+.exp.*.baidu.com'
+  - '*'