Merge branch 'main' into header

cmd/root.go

@@ -41,7 +41,8 @@ type skweezConf struct {
 	jsonOutput bool
 	targets    []string
 	urlFilter  []*regexp.Regexp
-	header     []string
+	userAgent  string
+  header     []string
 }
 
 var validWordRegex = regexp.MustCompile(`^[a-zA-Z0-9]+.*[a-zA-Z0-9]$`)
@@ -73,8 +74,10 @@ crawl websites to generate word lists.`,
 		handleErr(err, false)
 		paramJsonOutput, err := cmd.LocalFlags().GetBool("json")
 		handleErr(err, false)
-		paramHeader, err := cmd.LocalFlags().GetStringSlice("header")
+		paramUserAgent, err := cmd.LocalFlags().GetString("user-agent")
 		handleErr(err, false)
+    paramHeader, err := cmd.LocalFlags().GetStringSlice("header")
+    handleErr(err, false)
 		// sanitize scope param
 		sanitizedScope := []string{}
 		for _, element := range paramScope {
@@ -109,6 +112,7 @@ crawl websites to generate word lists.`,
 			noFilter:   paramNoFilter,
 			jsonOutput: paramJsonOutput,
 			targets:    preparedTargets,
+      userAgent:  paramUserAgent,
 			header:     paramHeader,
 		}
 		run(config)
@@ -129,6 +133,7 @@ func init() {
 	rootCmd.Flags().Bool("no-filter", false, "Do not filter out strings that don't match the regex to check if it looks like a valid word (starts and ends with alphanumeric letter, anything else in between). Also ignores --min-word-length and --max-word-length")
 	rootCmd.Flags().Bool("json", false, "Write words + counts in a json file. Requires --output/-o")
 	rootCmd.Flags().Bool("debug", false, "Enable Debug output")
+  rootCmd.Flags().StringP("user-agent", "a", "", "Set custom user-agent")
 	rootCmd.Flags().StringSlice("header", []string{}, "Additional header")
 }
 
@@ -160,6 +165,9 @@ func initColly(config *skweezConf) *colly.Collector {
 		colly.AllowedDomains(config.scope...),
 		colly.URLFilters(config.urlFilter...),
 	)
+	if config.userAgent != "" {
+		c.UserAgent = config.userAgent
+	}
 	c.AllowURLRevisit = false
 	return c
 }