Skip to content

Commit

Permalink
Pull request 1751: imp-querylog
Browse files Browse the repository at this point in the history 
Merge in DNS/adguard-home from imp-querylog to master

Squashed commit of the following:

commit 40b88f9
Merge: fcfe40b bb80a7c
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Feb 27 17:14:34 2023 +0300

    Merge branch 'master' into imp-querylog

commit fcfe40b
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Mon Feb 27 17:13:45 2023 +0300

    querylog: imp docs, names

commit 21722c6
Author: Ainar Garipov <A.Garipov@AdGuard.COM>
Date:   Wed Feb 22 20:28:49 2023 +0300

    querylog: fix race; refactor
  • Loading branch information
@ainar-g
ainar-g committed Feb 27, 2023
1 parent bb80a7c commit a772212
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 126 deletions.
67 changes: 28 additions & 39 deletions internal/querylog/decode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,55 +247,20 @@ var resultHandlers = map[string]logEntryHandler{
}

func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
var vToken json.Token
switch key {
case "FilterListID":
vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}

return
}

if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}

ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if n, ok := vToken.(json.Number); ok {
ent.Result.Rules[i].FilterListID, _ = n.Int64()
}
case "IP":
vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}

return
}

if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}

ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if ipStr, ok := vToken.(string); ok {
ent.Result.Rules[i].IP = net.ParseIP(ipStr)
}
case "Text":
vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}

return
}

if len(ent.Result.Rules) < i+1 {
ent.Result.Rules = append(ent.Result.Rules, &filtering.ResultRule{})
}

ent.Result.Rules, vToken = decodeVTokenAndAddRule(key, i, dec, ent.Result.Rules)
if s, ok := vToken.(string); ok {
ent.Result.Rules[i].Text = s
}
Expand All @@ -304,6 +269,30 @@ func decodeResultRuleKey(key string, i int, dec *json.Decoder, ent *logEntry) {
}
}

func decodeVTokenAndAddRule(
key string,
i int,
dec *json.Decoder,
rules []*filtering.ResultRule,
) (newRules []*filtering.ResultRule, vToken json.Token) {
newRules = rules

vToken, err := dec.Token()
if err != nil {
if err != io.EOF {
log.Debug("decodeResultRuleKey %s err: %s", key, err)
}

return newRules, nil
}

if len(rules) < i+1 {
newRules = append(newRules, &filtering.ResultRule{})
}

return newRules, vToken
}

func decodeResultRules(dec *json.Decoder, ent *logEntry) {
for {
delimToken, err := dec.Token()
Expand Down
3 changes: 0 additions & 3 deletions internal/querylog/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,7 @@ func (l *queryLog) handleQueryLog(w http.ResponseWriter, r *http.Request) {
return
}

// search for the log entries
entries, oldest := l.search(params)

// convert log entries to JSON
data := l.entriesToJSON(entries, oldest)

_ = aghhttp.WriteJSONResponse(w, r, data)
Expand Down
60 changes: 14 additions & 46 deletions internal/querylog/json.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package querylog

import (
"fmt"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -117,7 +116,7 @@ func (l *queryLog) setMsgData(entry *logEntry, jsonEntry jobject) {
// it from there as well.
jsonEntry["answer_dnssec"] = entry.AuthenticatedData || msg.AuthenticatedData

if a := answerToMap(msg); a != nil {
if a := answerToJSON(msg); a != nil {
jsonEntry["answer"] = a
}
}
Expand All @@ -136,7 +135,7 @@ func (l *queryLog) setOrigAns(entry *logEntry, jsonEntry jobject) {
return
}

if a := answerToMap(orig); a != nil {
if a := answerToJSON(orig); a != nil {
jsonEntry["original_answer"] = a
}
}
Expand All @@ -159,55 +158,24 @@ type dnsAnswer struct {
TTL uint32 `json:"ttl"`
}

func answerToMap(a *dns.Msg) (answers []*dnsAnswer) {
if a == nil || len(a.Answer) == 0 {
// answerToJSON converts the answer records of msg, if any, to their JSON form.
func answerToJSON(msg *dns.Msg) (answers []*dnsAnswer) {
if msg == nil || len(msg.Answer) == 0 {
return nil
}

answers = make([]*dnsAnswer, 0, len(a.Answer))
for _, k := range a.Answer {
header := k.Header()
answer := &dnsAnswer{
answers = make([]*dnsAnswer, 0, len(msg.Answer))
for _, rr := range msg.Answer {
header := rr.Header()
a := &dnsAnswer{
Type: dns.TypeToString[header.Rrtype],
TTL: header.Ttl,
// Remove the header string from the answer value since it's mostly
// unnecessary in the log.
Value: strings.TrimPrefix(rr.String(), header.String()),
TTL: header.Ttl,
}

// Some special treatment for some well-known types.
//
// TODO(a.garipov): Consider just calling String() for everyone
// instead.
switch v := k.(type) {
case nil:
// Probably unlikely, but go on.
case *dns.A:
answer.Value = v.A.String()
case *dns.AAAA:
answer.Value = v.AAAA.String()
case *dns.MX:
answer.Value = fmt.Sprintf("%v %v", v.Preference, v.Mx)
case *dns.CNAME:
answer.Value = v.Target
case *dns.NS:
answer.Value = v.Ns
case *dns.SPF:
answer.Value = strings.Join(v.Txt, "\n")
case *dns.TXT:
answer.Value = strings.Join(v.Txt, "\n")
case *dns.PTR:
answer.Value = v.Ptr
case *dns.SOA:
answer.Value = fmt.Sprintf("%v %v %v %v %v %v %v", v.Ns, v.Mbox, v.Serial, v.Refresh, v.Retry, v.Expire, v.Minttl)
case *dns.CAA:
answer.Value = fmt.Sprintf("%v %v \"%v\"", v.Flag, v.Tag, v.Value)
case *dns.HINFO:
answer.Value = fmt.Sprintf("\"%v\" \"%v\"", v.Cpu, v.Os)
case *dns.RRSIG:
answer.Value = fmt.Sprintf("%v %v %v %v %v %v %v %v %v", dns.TypeToString[v.TypeCovered], v.Algorithm, v.Labels, v.OrigTtl, v.Expiration, v.Inception, v.KeyTag, v.SignerName, v.Signature)
default:
answer.Value = v.String()
}

answers = append(answers, answer)
answers = append(answers, a)
}

return answers
Expand Down
10 changes: 9 additions & 1 deletion internal/querylog/qlog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ type queryLog struct {

// bufferLock protects buffer.
bufferLock sync.RWMutex
// buffer contains recent log entries.
// buffer contains recent log entries. The entries in this buffer must not
// be modified.
buffer []*logEntry

fileFlushLock sync.Mutex // synchronize a file-flushing goroutine and main thread
Expand Down Expand Up @@ -100,6 +101,13 @@ type logEntry struct {
AuthenticatedData bool `json:"AD,omitempty"`
}

// shallowClone returns a shallow clone of e.
func (e *logEntry) shallowClone() (clone *logEntry) {
cloneVal := *e

return &cloneVal
}

func (l *queryLog) Start() {
if l.conf.HTTPRegister != nil {
l.initWeb()
Expand Down
6 changes: 4 additions & 2 deletions internal/querylog/search.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ func (l *queryLog) searchMemory(params *searchParams, cache clientCache) (entrie
// Go through the buffer in the reverse order, from newer to older.
var err error
for i := len(l.buffer) - 1; i >= 0; i-- {
e := l.buffer[i]
// A shallow clone is enough, since the only thing that this loop
// modifies is the client field.
e := l.buffer[i].shallowClone()

e.client, err = l.client(e.ClientID, e.IP.String(), cache)
if err != nil {
Expand Down Expand Up @@ -130,7 +132,7 @@ func (l *queryLog) search(params *searchParams) (entries []*logEntry, oldest tim
// searchFiles looks up log records from all log files. It optionally uses the
// client cache, if provided. searchFiles does not scan more than
// maxFileScanEntries so callers may need to call it several times to get all
// results. oldset and total are the time of the oldest processed entry and the
// results. oldest and total are the time of the oldest processed entry and the
// total number of processed entries, including discarded ones, correspondingly.
func (l *queryLog) searchFiles(
params *searchParams,
Expand Down
85 changes: 51 additions & 34 deletions internal/querylog/searchcriterion.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package querylog

import (
"fmt"
"strings"

"github.com/AdguardTeam/AdGuardHome/internal/filtering"
Expand Down Expand Up @@ -118,7 +119,7 @@ func (c *searchCriterion) match(entry *logEntry) bool {
case ctTerm:
return c.ctDomainOrClientCase(entry)
case ctFilteringStatus:
return c.ctFilteringStatusCase(entry.Result)
return c.ctFilteringStatusCase(entry.Result.Reason, entry.Result.IsFiltered)
}

return false
Expand All @@ -141,54 +142,70 @@ func (c *searchCriterion) ctDomainOrClientCase(e *logEntry) bool {
return ctDomainOrClientCaseNonStrict(c.value, c.asciiVal, clientID, name, host, ip)
}

func (c *searchCriterion) ctFilteringStatusCase(res filtering.Result) bool {
// ctFilteringStatusCase returns true if the result matches the value.
func (c *searchCriterion) ctFilteringStatusCase(
reason filtering.Reason,
isFiltered bool,
) (matched bool) {
switch c.value {
case filteringStatusAll:
return true

case filteringStatusFiltered:
return res.IsFiltered ||
res.Reason.In(
filtering.NotFilteredAllowList,
filtering.Rewritten,
filtering.RewrittenAutoHosts,
filtering.RewrittenRule,
)

case filteringStatusBlocked:
return res.IsFiltered &&
res.Reason.In(filtering.FilteredBlockList, filtering.FilteredBlockedService)

case filteringStatusBlockedService:
return res.IsFiltered && res.Reason == filtering.FilteredBlockedService

case filteringStatusBlockedParental:
return res.IsFiltered && res.Reason == filtering.FilteredParental

case filteringStatusBlockedSafebrowsing:
return res.IsFiltered && res.Reason == filtering.FilteredSafeBrowsing

case
filteringStatusBlocked,
filteringStatusBlockedParental,
filteringStatusBlockedSafebrowsing,
filteringStatusBlockedService,
filteringStatusFiltered,
filteringStatusSafeSearch:
return isFiltered && c.isFilteredWithReason(reason)
case filteringStatusWhitelisted:
return res.Reason == filtering.NotFilteredAllowList

return reason == filtering.NotFilteredAllowList
case filteringStatusRewritten:
return res.Reason.In(
return reason.In(
filtering.Rewritten,
filtering.RewrittenAutoHosts,
filtering.RewrittenRule,
)

case filteringStatusSafeSearch:
return res.IsFiltered && res.Reason == filtering.FilteredSafeSearch

case filteringStatusProcessed:
return !res.Reason.In(
return !reason.In(
filtering.FilteredBlockList,
filtering.FilteredBlockedService,
filtering.NotFilteredAllowList,
)

default:
return false
}
}

// isFilteredWithReason returns true if reason matches the criterion value.
// c.value must be one of:
//
// - filteringStatusBlocked
// - filteringStatusBlockedParental
// - filteringStatusBlockedSafebrowsing
// - filteringStatusBlockedService
// - filteringStatusFiltered
// - filteringStatusSafeSearch
func (c *searchCriterion) isFilteredWithReason(reason filtering.Reason) (matched bool) {
switch c.value {
case filteringStatusBlocked:
return reason.In(filtering.FilteredBlockList, filtering.FilteredBlockedService)
case filteringStatusBlockedParental:
return reason == filtering.FilteredParental
case filteringStatusBlockedSafebrowsing:
return reason == filtering.FilteredSafeBrowsing
case filteringStatusBlockedService:
return reason == filtering.FilteredBlockedService
case filteringStatusFiltered:
return reason.In(
filtering.NotFilteredAllowList,
filtering.Rewritten,
filtering.RewrittenAutoHosts,
filtering.RewrittenRule,
)
case filteringStatusSafeSearch:
return reason == filtering.FilteredSafeSearch
default:
panic(fmt.Errorf("unexpected value %q", c.value))
}
}
2 changes: 1 addition & 1 deletion scripts/make/go-lint.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ run_linter "$GO" vet ./...
run_linter govulncheck ./...

# Apply more lax standards to the code we haven't properly refactored yet.
run_linter gocyclo --over 17 ./internal/querylog/
run_linter gocyclo --over 14 ./internal/querylog/
run_linter gocyclo --over 13\
./internal/dhcpd\
./internal/filtering/\
Expand Down

0 comments on commit a772212

Please sign in to comment.