From 07f58be3b878d399c02edad1597178a93e4227f1 Mon Sep 17 00:00:00 2001
From: Jason Ertel <jason.ertel@securityonionsolutions.com>
Date: Fri, 21 Oct 2022 10:46:48 -0400
Subject: [PATCH] Remove sortby clause when querying related case events (bug
 in ES?)

---
 server/modules/elastic/elasticcasestore.go    | 20 +++++++++++-
 .../modules/elastic/elasticcasestore_test.go  | 32 ++++++++++++++++++-
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/server/modules/elastic/elasticcasestore.go b/server/modules/elastic/elasticcasestore.go
index 142f8bc1..f687ef65 100644
--- a/server/modules/elastic/elasticcasestore.go
+++ b/server/modules/elastic/elasticcasestore.go
@@ -18,6 +18,7 @@ import (
   "github.com/security-onion-solutions/securityonion-soc/server"
   "github.com/security-onion-solutions/securityonion-soc/web"
   "regexp"
+  "sort"
   "strconv"
   "time"
 )
@@ -524,13 +525,30 @@ func (store *ElasticCasestore) GetRelatedEvents(ctx context.Context, caseId stri
   err = store.validateId(caseId, "caseId")
   if err == nil {
     events = make([]*model.RelatedEvent, 0)
-    query := fmt.Sprintf(`_index:"%s" AND %skind:"related" AND %srelated.caseId:"%s" | sortby %srelated.fields.timestamp^`, store.index, store.schemaPrefix, store.schemaPrefix, caseId, store.schemaPrefix)
+    // JBE 10/20/2022: Remove sortby due to issue with Elastic 8.4 causing incompatible sort field types
+    //  | sortby %srelated.fields.timestamp^
+    query := fmt.Sprintf(`_index:"%s" AND %skind:"related" AND %srelated.caseId:"%s"`, store.index, store.schemaPrefix, store.schemaPrefix, caseId)
     var objects []interface{}
     objects, err = store.getAll(ctx, query, store.maxAssociations)
     if err == nil {
       for _, obj := range objects {
         events = append(events, obj.(*model.RelatedEvent))
       }
+
+      // JBE 10/20/2022: Manually sort the related events by the timestamp field, in ascending order. This can remain
+      // in place even if the above ES issue is resolved.
+      sort.Slice(events, func(a, b int) bool {
+        if ts_a, ts_a_exists := events[a].Fields["timestamp"]; ts_a_exists {
+          if ts_a_typed, ts_a_correct_type := ts_a.(time.Time); ts_a_correct_type {
+            if ts_b, ts_b_exists := events[b].Fields["timestamp"]; ts_b_exists {
+              if ts_b_typed, ts_b_correct_type := ts_b.(time.Time); ts_b_correct_type {
+                return ts_a_typed.Before(ts_b_typed)
+              }
+            }
+          }
+        }
+        return false
+      })
     }
   }
   return events, err
diff --git a/server/modules/elastic/elasticcasestore_test.go b/server/modules/elastic/elasticcasestore_test.go
index 7e55d6bf..5a0d9dda 100644
--- a/server/modules/elastic/elasticcasestore_test.go
+++ b/server/modules/elastic/elasticcasestore_test.go
@@ -17,6 +17,7 @@ import (
 	"github.com/security-onion-solutions/securityonion-soc/web"
 	"github.com/stretchr/testify/assert"
 	"testing"
+	"time"
 )
 
 func TestInit(tester *testing.T) {
@@ -966,18 +967,47 @@ func TestGetRelatedEvents(tester *testing.T) {
 	fakeEventStore := server.NewFakeEventstore()
 	store.server.Eventstore = fakeEventStore
 	ctx := context.WithValue(context.Background(), web.ContextKeyRequestorId, "myRequestorId")
-	query := `_index:"myIndex" AND so_kind:"related" AND so_related.caseId:"myCaseId" | sortby so_related.fields.timestamp^`
+	// JBE: 10/20/2022 - Remove sortby and perform it manually due to ES issue with flattened fields
+	//query := `_index:"myIndex" AND so_kind:"related" AND so_related.caseId:"myCaseId" | sortby so_related.fields.timestamp^`
+	query := `_index:"myIndex" AND so_kind:"related" AND so_related.caseId:"myCaseId"`
 	eventPayload := make(map[string]interface{})
 	eventPayload["so_kind"] = "related"
 	elasticEvent := &model.EventRecord{
 		Payload: eventPayload,
 	}
 	fakeEventStore.SearchResults[0].Events = append(fakeEventStore.SearchResults[0].Events, elasticEvent)
+
+	// Add a related event with a timestamp field
+	timeA, _ := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
+	eventPayloadWithTimestamp := make(map[string]interface{})
+	eventPayloadWithTimestamp["so_kind"] = "related"
+	eventPayloadWithTimestamp["so_related.fields.timestamp"] = timeA
+	elasticEventWithTimestamp := &model.EventRecord{
+		Payload: eventPayloadWithTimestamp,
+	}
+	fakeEventStore.SearchResults[0].Events = append(fakeEventStore.SearchResults[0].Events, elasticEventWithTimestamp)
+
+	// Add another related event with an earlier timestamp field
+	timeB, _ := time.Parse(time.RFC3339, "2006-01-01T15:04:05Z")
+	eventPayloadWithTimestamp2 := make(map[string]interface{})
+	eventPayloadWithTimestamp2["so_kind"] = "related"
+	eventPayloadWithTimestamp2["so_related.fields.timestamp"] = timeB
+	elasticEventWithTimestamp2 := &model.EventRecord{
+		Payload: eventPayloadWithTimestamp2,
+	}
+	fakeEventStore.SearchResults[0].Events = append(fakeEventStore.SearchResults[0].Events, elasticEventWithTimestamp2)
+
 	obj, err := store.GetRelatedEvents(ctx, "myCaseId")
 	assert.NoError(tester, err)
 	assert.Len(tester, fakeEventStore.InputSearchCriterias, 1)
 	assert.Equal(tester, query, fakeEventStore.InputSearchCriterias[0].RawQuery)
 	assert.NotNil(tester, obj)
+
+	// Ensure manual sorting functions as expected (only has effect if the sortby claused is removed)
+	assert.Len(tester, obj, 3)
+	assert.Nil(tester, obj[0].Fields["timestamp"])
+	assert.Equal(tester, timeB, obj[1].Fields["timestamp"])
+	assert.Equal(tester, timeA, obj[2].Fields["timestamp"])
 }
 
 func TestDeleteRelatedEvent(tester *testing.T) {