add support for comments in files (#224)

* add support for comments in files

* update lint action

* fix lint errs

.github/workflows/lint-test.yml

@@ -9,18 +9,10 @@ on:
 
 jobs:
   lint:
-    name: Lint Test
+    name: "Lint"
+    if: "${{ !endsWith(github.actor, '[bot]') }}"
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-      - name: Set up Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.21.x
-      - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.3.0
-        with:
-          version: latest
-          args: --timeout 5m
-          working-directory: .
+      - uses: actions/checkout@v4
+      - uses: projectdiscovery/actions/setup/go@v1
+      - uses: projectdiscovery/actions/golangci-lint@v1

ratelimit_var.go

@@ -77,7 +77,7 @@ func (rateLimitMap *RateLimitMap) Del(key string) error {
 
 // IsEmpty specifies if the underlying map is empty
 func (rateLimitMap *RateLimitMap) IsEmpty() bool {
-	return rateLimitMap.kv == nil || len(rateLimitMap.kv) == 0
+	return len(rateLimitMap.kv) == 0
 }
 
 // AsMap returns the internal map as reference - changes are allowed

runtime_map.go

@@ -60,7 +60,7 @@ func (runtimeMap *RuntimeMap) Del(key string) error {
 
 // IsEmpty specifies if the underlying map is empty
 func (runtimeMap *RuntimeMap) IsEmpty() bool {
-	return runtimeMap.kv == nil || len(runtimeMap.kv) == 0
+	return len(runtimeMap.kv) == 0
 }
 
 // AsMap returns the internal map as reference - changes are allowed

slice_common.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/pkg/errors"
 	fileutil "github.com/projectdiscovery/utils/file"
+	stringsutil "github.com/projectdiscovery/utils/strings"
 )
 
 var quotes = []rune{'"', '\'', '`'}
@@ -64,10 +65,10 @@ func ToStringSlice(value string, options Options) ([]string, error) {
 	}
 
 	addPartToResult := func(part string) {
+		if options.Normalize != nil {
+			part = options.Normalize(part)
+		}
 		if !options.IsEmpty(part) {
-			if options.Normalize != nil {
-				part = options.Normalize(part)
-			}
 			result = append(result, part)
 		}
 	}
@@ -120,13 +121,31 @@ func isFromFile(_ string) bool {
 }
 
 func normalizeTrailingParts(s string) string {
-	return strings.TrimSpace(s)
+	return stringsutil.NormalizeWithOptions(s,
+		stringsutil.NormalizeOptions{
+			StripComments: true,
+			TrimSpaces:    true,
+		},
+	)
 }
 
 func normalize(s string) string {
-	return strings.TrimSpace(strings.Trim(strings.TrimSpace(s), string(quotes)))
+	return stringsutil.NormalizeWithOptions(s,
+		stringsutil.NormalizeOptions{
+			StripComments: true,
+			TrimCutset:    string(quotes),
+			TrimSpaces:    true,
+		},
+	)
 }
 
 func normalizeLowercase(s string) string {
-	return strings.TrimSpace(strings.Trim(strings.TrimSpace(strings.ToLower(s)), string(quotes)))
+	return stringsutil.NormalizeWithOptions(s,
+		stringsutil.NormalizeOptions{
+			StripComments: true,
+			TrimCutset:    string(quotes),
+			TrimSpaces:    true,
+			Lowercase:     true,
+		},
+	)
 }

string_slice_test.go

@@ -86,7 +86,7 @@ func TestFileNormalizedStringSliceOptions(t *testing.T) {
 
 func TestFileStringSliceOptions(t *testing.T) {
 	filename := "test.txt"
-	_ = os.WriteFile(filename, []byte("value1,value2\nvalue3"), 0644)
+	_ = os.WriteFile(filename, []byte("# this is a comment\nvalue1,value2\nvalue3"), 0644)
 	defer os.RemoveAll(filename)
 
 	result, err := ToStringSlice(filename, FileStringSliceOptions)