add awesome search queries (#595)

* add awesome search queries

* update readme

* minor

* update lint workflow

* fix lint err

.github/workflows/lint-test.yml

@@ -8,21 +8,15 @@ on:
   workflow_dispatch:
 
 jobs:
-  lint:
-    name: Lint Test
+  lint-test:
+    if: "! endsWith(github.actor, '[bot]')"
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: 1.21.x
-
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: projectdiscovery/actions/setup/go@v1
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3.6.0
-        with:
-          version: latest
-          args: --timeout 5m
-          working-directory: .
+        uses: projectdiscovery/actions/golangci-lint@v1

README.md

@@ -76,6 +76,7 @@ Flags:
 INPUT:
    -q, -query string[]   search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')
    -e, -engine string[]  search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,criminalip,publicwww,hunterhow,google) (default shodan)
+   -asq, -awesome-search-queries string[]  use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')
 
 SEARCH-ENGINE:
    -s, -shodan string[]       search query for shodan (example: -shodan 'query.txt')

go.mod

@@ -1,6 +1,8 @@
 module github.com/projectdiscovery/uncover
 
-go 1.21
+go 1.21.3
+
+toolchain go1.23.2
 
 require (
 	github.com/hashicorp/golang-lru v0.5.4
@@ -88,6 +90,7 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/projectdiscovery/awesome-search-queries v0.0.0-20241111181020-ad0097bd63a1 // indirect
 	github.com/projectdiscovery/fastdialer v0.2.9 // indirect
 	github.com/projectdiscovery/hmap v0.0.64 // indirect
 	github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect

go.sum

@@ -183,6 +183,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
 github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/projectdiscovery/awesome-search-queries v0.0.0-20241111181020-ad0097bd63a1 h1:DOl0vtiugcqv7gs/8lcIlwcp1My60JqAl1Euzrw9q2w=
+github.com/projectdiscovery/awesome-search-queries v0.0.0-20241111181020-ad0097bd63a1/go.mod h1:nSovPcipgSx/EzAefF+iCfORolkKAuodiRWL3RCGHOM=
 github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ=
 github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss=
 github.com/projectdiscovery/fastdialer v0.2.9 h1:vDCqxVMCyUu3oVEizEK1K8K+CCcLkVDW3X2HfiWaVFA=

runner/options.go

@@ -1,11 +1,14 @@
 package runner
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
+	"slices"
 
 	"errors"
 
+	awesomesearchqueries "github.com/projectdiscovery/awesome-search-queries"
 	"github.com/projectdiscovery/goflags"
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/gologger/formatter"
@@ -25,35 +28,36 @@ var (
 
 // Options contains the configuration options for tuning the enumeration process.
 type Options struct {
-	Query              goflags.StringSlice
-	Engine             goflags.StringSlice
-	ConfigFile         string
-	ProviderFile       string
-	OutputFile         string
-	OutputFields       string
-	JSON               bool
-	Raw                bool
-	Limit              int
-	Silent             bool
-	Verbose            bool
-	NoColor            bool
-	Timeout            int
-	RateLimit          int
-	RateLimitMinute    int
-	Retries            int
-	Shodan             goflags.StringSlice
-	ShodanIdb          goflags.StringSlice
-	Fofa               goflags.StringSlice
-	Censys             goflags.StringSlice
-	Quake              goflags.StringSlice
-	Netlas             goflags.StringSlice
-	Hunter             goflags.StringSlice
-	ZoomEye            goflags.StringSlice
-	CriminalIP         goflags.StringSlice
-	Publicwww          goflags.StringSlice
-	HunterHow          goflags.StringSlice
-	Google             goflags.StringSlice
-	DisableUpdateCheck bool
+	Query                goflags.StringSlice
+	Engine               goflags.StringSlice
+	AwesomeSearchQueries goflags.StringSlice
+	ConfigFile           string
+	ProviderFile         string
+	OutputFile           string
+	OutputFields         string
+	JSON                 bool
+	Raw                  bool
+	Limit                int
+	Silent               bool
+	Verbose              bool
+	NoColor              bool
+	Timeout              int
+	RateLimit            int
+	RateLimitMinute      int
+	Retries              int
+	Shodan               goflags.StringSlice
+	ShodanIdb            goflags.StringSlice
+	Fofa                 goflags.StringSlice
+	Censys               goflags.StringSlice
+	Quake                goflags.StringSlice
+	Netlas               goflags.StringSlice
+	Hunter               goflags.StringSlice
+	ZoomEye              goflags.StringSlice
+	CriminalIP           goflags.StringSlice
+	Publicwww            goflags.StringSlice
+	HunterHow            goflags.StringSlice
+	Google               goflags.StringSlice
+	DisableUpdateCheck   bool
 }
 
 // ParseOptions parses the command line flags provided by a user
@@ -65,6 +69,7 @@ func ParseOptions() *Options {
 	flagSet.CreateGroup("input", "Input",
 		flagSet.StringSliceVarP(&options.Query, "query", "q", nil, "search query, supports: stdin,file,config input (example: -q 'example query', -q 'query.txt')", goflags.FileStringSliceOptions),
 		flagSet.StringSliceVarP(&options.Engine, "engine", "e", nil, "search engine to query (shodan,shodan-idb,fofa,censys,quake,hunter,zoomeye,netlas,publicwww,criminalip,hunterhow,google) (default shodan)", goflags.FileNormalizedStringSliceOptions),
+		flagSet.StringSliceVarP(&options.AwesomeSearchQueries, "awesome-search-queries", "asq", nil, "use awesome search queries to discover exposed assets on the internet (example: -asq 'jira')", goflags.FileStringSliceOptions),
 	)
 
 	flagSet.CreateGroup("search-engine", "Search-Engine",
@@ -165,6 +170,12 @@ func ParseOptions() *Options {
 		}
 	}
 
+	if len(options.AwesomeSearchQueries) > 0 {
+		if err := options.useAwesomeSearchQueries(options.AwesomeSearchQueries); err != nil {
+			gologger.Fatal().Msgf("could not use awesome search queries: %s\n", err)
+		}
+	}
+
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
 	if err := options.validateOptions(); err != nil {
@@ -250,8 +261,15 @@ func versionCallback() {
 
 func appendQuery(options *Options, name string, queries ...string) {
 	if len(queries) > 0 {
-		options.Engine = append(options.Engine, name)
-		options.Query = append(options.Query, queries...)
+		if !slices.Contains(options.Engine, name) {
+			options.Engine = append(options.Engine, name)
+		}
+
+		for _, query := range queries {
+			if !slices.Contains(options.Query, query) {
+				options.Query = append(options.Query, query)
+			}
+		}
 	}
 }
 
@@ -269,3 +287,27 @@ func appendAllQueries(options *Options) {
 	appendQuery(options, "hunterhow", options.HunterHow...)
 	appendQuery(options, "google", options.Google...)
 }
+
+func (options *Options) useAwesomeSearchQueries(awesomeSearchQueries []string) error {
+	data, err := awesomesearchqueries.GetQueries()
+	if err != nil {
+		return err
+	}
+
+	var queries []awesomesearchqueries.Query
+	if err := json.Unmarshal(data, &queries); err != nil {
+		return err
+	}
+
+	// TODO: This is ugly. Improve this by adding direct query support in awesome-search-queries.
+	for _, query := range awesomeSearchQueries {
+		for _, engine := range queries {
+			if engine.Name == query {
+				for _, engine := range engine.Engines {
+					appendQuery(options, engine.Platform, engine.Queries...)
+				}
+			}
+		}
+	}
+	return nil
+}

sources/agent/fofa/fofa.go

@@ -83,7 +83,7 @@ func (agent *Agent) query(URL string, session *sources.Session, fofaRequest *Fof
 		return nil
 	}
 	if fofaResponse.Error {
-		results <- sources.Result{Source: agent.Name(), Error: fmt.Errorf(fofaResponse.ErrMsg)}
+		results <- sources.Result{Source: agent.Name(), Error: fmt.Errorf("%s", fofaResponse.ErrMsg)}
 		return nil
 	}