Merge pull request #14961 from smartcontractkit/fix-datawords-search

Added dynamic fields indexing

core/services/relay/evm/chain_reader.go

@@ -302,7 +302,7 @@ func (cr *chainReader) addEvent(contractName, eventName string, a abi.ABI, chain
 		return fmt.Errorf("%w: event %q doesn't exist", commontypes.ErrInvalidConfig, chainReaderDefinition.ChainSpecificName)
 	}
 
-	indexedAsUnIndexedABITypes, indexedTopicsCodecTypes, eventDWs := getEventTypes(event)
+	indexedAsUnIndexedABITypes, indexedTopicsCodecTypes, dwsDetails := getEventTypes(event)
 	if err := indexedTopicsCodecTypes.Init(); err != nil {
 		return err
 	}
@@ -337,7 +337,7 @@ func (cr *chainReader) addEvent(contractName, eventName string, a abi.ABI, chain
 		maps.Copy(codecModifiers, topicsModifiers)
 
 		// TODO BCFR-44 no dw modifier for now
-		dataWordsDetails, dWSCodecTypeInfo, initDWQueryingErr := cr.initDWQuerying(contractName, eventName, eventDWs, eventDefinitions.GenericDataWordDetails)
+		dataWordsDetails, dWSCodecTypeInfo, initDWQueryingErr := cr.initDWQuerying(contractName, eventName, dwsDetails, eventDefinitions.GenericDataWordDetails)
 		if initDWQueryingErr != nil {
 			return fmt.Errorf("failed to init dw querying for event: %q, err: %w", eventName, initDWQueryingErr)
 		}
@@ -473,71 +473,94 @@ func (cr *chainReader) addDecoderDef(contractName, itemType string, outputs abi.
 func getEventTypes(event abi.Event) ([]abi.Argument, types.CodecEntry, map[string]read.DataWordDetail) {
 	indexedAsUnIndexedTypes := make([]abi.Argument, 0, types.MaxTopicFields)
 	indexedTypes := make([]abi.Argument, 0, len(event.Inputs))
-	dataWords := make(map[string]read.DataWordDetail)
-	var dwIndex int
-
 	for _, input := range event.Inputs {
-		if !input.Indexed {
-			dwIndex = calculateFieldDWIndex(input.Type, event.Name+"."+input.Name, dataWords, dwIndex)
-			continue
+		if input.Indexed {
+			indexedAsUnIndexed := input
+			indexedAsUnIndexed.Indexed = false
+			// when presenting the filter off-chain, the caller will provide the unHashed version of the input and CR will hash topics when needed.
+			indexedAsUnIndexedTypes = append(indexedAsUnIndexedTypes, indexedAsUnIndexed)
+			indexedTypes = append(indexedTypes, input)
 		}
+	}
+
+	return indexedAsUnIndexedTypes, types.NewCodecEntry(indexedTypes, nil, nil), getDWIndexesWithTypes(event.Name, event.Inputs)
+}
 
-		indexedAsUnIndexed := input
-		indexedAsUnIndexed.Indexed = false
-		// when presenting the filter off-chain, the caller will provide the unHashed version of the input and CR will hash topics when needed.
-		indexedAsUnIndexedTypes = append(indexedAsUnIndexedTypes, indexedAsUnIndexed)
-		indexedTypes = append(indexedTypes, input)
+func getDWIndexesWithTypes(eventName string, eventInputs abi.Arguments) map[string]read.DataWordDetail {
+	var dwIndexOffset int
+	dataWords := make(map[string]read.DataWordDetail)
+	dynamicQueue := make([]abi.Argument, 0)
+
+	for _, input := range eventInputs.NonIndexed() {
+		// each dynamic field has an extra field that stores the dwIndexOffset that points to the start of the dynamic data.
+		if isDynamic(input.Type) {
+			dynamicQueue = append(dynamicQueue, input)
+			dwIndexOffset = dwIndexOffset + 1
+		} else {
+			dwIndexOffset = processDWStaticField(input.Type, eventName+"."+input.Name, dataWords, dwIndexOffset)
+		}
 	}
 
-	return indexedAsUnIndexedTypes, types.NewCodecEntry(indexedTypes, nil, nil), dataWords
+	processDWDynamicFields(eventName, dataWords, dynamicQueue, dwIndexOffset)
+	return dataWords
 }
 
-// calculateFieldDWIndex recursively calculates the indices of all static unindexed fields in the event
-// and calculates the offset for all unsearchable / dynamic fields.
-func calculateFieldDWIndex(fieldType abi.Type, fieldPath string, dataWords map[string]read.DataWordDetail, index int) int {
+func isDynamic(fieldType abi.Type) bool {
 	switch fieldType.T {
+	case abi.StringTy, abi.SliceTy, abi.BytesTy:
+		return true
 	case abi.TupleTy:
-		// each dynamic tuple(tuple which has at least 1 dynamic field) has a 32 byte header that points to the first dynamic field
-		if isDynamic(fieldType) {
-			index++
+		// If one element in a struct is dynamic, the whole struct is treated as dynamic.
+		for _, elem := range fieldType.TupleElems {
+			if isDynamic(*elem) {
+				return true
+			}
 		}
+	}
+	return false
+}
+
+func processDWStaticField(fieldType abi.Type, parentFieldPath string, dataWords map[string]read.DataWordDetail, index int) int {
+	switch fieldType.T {
+	case abi.TupleTy:
 		// Recursively process tuple elements
 		for i, tupleElem := range fieldType.TupleElems {
 			fieldName := fieldType.TupleRawNames[i]
-			fullFieldPath := fmt.Sprintf("%s.%s", fieldPath, fieldName)
-			index = calculateFieldDWIndex(*tupleElem, fullFieldPath, dataWords, index)
+			fullFieldPath := fmt.Sprintf("%s.%s", parentFieldPath, fieldName)
+			index = processDWStaticField(*tupleElem, fullFieldPath, dataWords, index)
 		}
 		return index
 	case abi.ArrayTy:
 		// Static arrays are not searchable, however, we can reliably calculate their size so that the fields
 		// after them can be searched.
 		return index + fieldType.Size
 	default:
-		// if field is dynamic and not a tuple all the sequential fields data word indexes aren't deterministic
-		if !isDynamic(fieldType) {
-			dataWords[fieldPath] = read.DataWordDetail{
-				Index:    index,
-				Argument: abi.Argument{Type: fieldType},
-			}
+		dataWords[parentFieldPath] = read.DataWordDetail{
+			Index:    index,
+			Argument: abi.Argument{Type: fieldType},
 		}
-
 		return index + 1
 	}
 }
 
-func isDynamic(fieldType abi.Type) bool {
-	switch fieldType.T {
-	case abi.StringTy, abi.SliceTy, abi.BytesTy:
-		return true
-	case abi.TupleTy:
-		// If one element in a struct is dynamic, the whole struct is treated as dynamic.
-		for _, elem := range fieldType.TupleElems {
-			if isDynamic(*elem) {
-				return true
+// processDWDynamicFields indexes static fields in dynamic structs.
+// These fields come first after the static fields in the event encoding, so we can calculate their indices.
+func processDWDynamicFields(eventName string, dataWords map[string]read.DataWordDetail, dynamicQueue []abi.Argument, dwIndex int) {
+	for _, arg := range dynamicQueue {
+		switch arg.Type.T {
+		case abi.TupleTy:
+			for i, tupleElem := range arg.Type.TupleElems {
+				// any field after a dynamic field can't be predictably indexed.
+				if isDynamic(*tupleElem) {
+					return
+				}
+				dwIndex = processDWStaticField(*tupleElem, fmt.Sprintf("%s.%s.%s", eventName, arg.Name, arg.Type.TupleRawNames[i]), dataWords, dwIndex)
 			}
+		default:
+			// exit if we see a dynamic field, as we can't predict the index of fields after it.
+			return
 		}
 	}
-	return false
 }
 
 // ConfirmationsFromConfig maps chain agnostic confidence levels defined in config to predefined EVM finality.

core/services/relay/evm/evmtesting/chain_components_interface_tester.go

@@ -147,6 +147,7 @@ func (it *EVMChainComponentsInterfaceTester[T]) Setup(t T) {
 								"OracleID": {Name: "oracleId"},
 								// this is just to illustrate an example, generic names shouldn't really be formatted like this since other chains might not store it in the same way
 								"NestedStaticStruct.Inner.IntVal": {Name: "nestedStaticStruct.Inner.IntVal"},
+								"NestedDynamicStruct.FixedBytes":  {Name: "nestedDynamicStruct.FixedBytes"},
 								"BigField":                        {Name: "bigField"},
 							},
 						},

core/services/relay/evm/evmtesting/run_tests.go

@@ -223,6 +223,26 @@ func RunContractReaderInLoopTests[T TestingT[T]](t T, it ChainComponentsInterfac
 		}, it.MaxWaitTimeForEvents(), time.Millisecond*10)
 	})
 
+	t.Run("Filtering can be done on data words using value comparator on a static field in a dynamic struct that is the first dynamic field", func(t T) {
+		ts := &TestStruct{}
+		assert.Eventually(t, func() bool {
+			sequences, err := cr.QueryKey(ctx, boundContract, query.KeyFilter{Key: EventName, Expressions: []query.Expression{
+				query.Comparator("OracleID",
+					primitives.ValueComparator{
+						Value:    uint8(ts2.OracleID),
+						Operator: primitives.Eq,
+					}),
+				query.Comparator("NestedDynamicStruct.FixedBytes",
+					primitives.ValueComparator{
+						Value:    ts2.NestedDynamicStruct.FixedBytes,
+						Operator: primitives.Eq,
+					}),
+			},
+			}, query.LimitAndSort{}, ts)
+			return err == nil && len(sequences) == 1 && reflect.DeepEqual(&ts2, sequences[0].Data)
+		}, it.MaxWaitTimeForEvents(), time.Millisecond*10)
+	})
+
 	t.Run("Filtering can be done on data words using value comparators on fields that require manual index input", func(t T) {
 		empty12Bytes := [12]byte{}
 		val1, val2, val3, val4 := uint32(1), uint32(2), uint32(3), uint64(4)