ccip - graceful shutdown of chain readers and contract writers (#14656)

* close resources with wrapped oracle

* changeset

* fix changeset

* rm func

* run custom Closing after oracle Close

* lint fix

* use io.Closer

* use services.CloseAll

.changeset/moody-trains-grow.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+#added graceful shutdown for ccip oracles

core/capabilities/ccip/delegate.go

@@ -21,12 +21,13 @@ import (
 
 	ragep2ptypes "github.com/smartcontractkit/libocr/ragep2p/types"
 
-	"github.com/smartcontractkit/chainlink-ccip/pkg/consts"
-	ccipreaderpkg "github.com/smartcontractkit/chainlink-ccip/pkg/reader"
 	"github.com/smartcontractkit/chainlink-common/pkg/sqlutil"
 	"github.com/smartcontractkit/chainlink-common/pkg/types"
 	"github.com/smartcontractkit/chainlink-common/pkg/types/query/primitives"
 
+	"github.com/smartcontractkit/chainlink-ccip/pkg/consts"
+	ccipreaderpkg "github.com/smartcontractkit/chainlink-ccip/pkg/reader"
+
 	chainsel "github.com/smartcontractkit/chain-selectors"
 
 	cciptypes "github.com/smartcontractkit/chainlink-common/pkg/types/ccipocr3"

core/capabilities/ccip/oraclecreator/plugin.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"math/big"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
+	chainsel "github.com/smartcontractkit/chain-selectors"
 	"github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper"
 
 	"github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/ccipevm"
@@ -20,8 +22,6 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml"
 
-	chainsel "github.com/smartcontractkit/chain-selectors"
-
 	"github.com/smartcontractkit/libocr/commontypes"
 	libocr3 "github.com/smartcontractkit/libocr/offchainreporting2plus"
 	"github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types"
@@ -203,7 +203,15 @@ func (i *pluginOracleCreator) Create(donID uint32, config cctypes.OCR3ConfigWith
 	if err != nil {
 		return nil, err
 	}
-	return oracle, nil
+
+	closers := make([]io.Closer, 0, len(contractReaders)+len(chainWriters))
+	for _, cr := range contractReaders {
+		closers = append(closers, cr)
+	}
+	for _, cw := range chainWriters {
+		closers = append(closers, cw)
+	}
+	return newWrappedOracle(oracle, closers), nil
 }
 
 func (i *pluginOracleCreator) createFactoryAndTransmitter(

core/capabilities/ccip/oraclecreator/wrapped_oracle.go

@@ -0,0 +1,42 @@
+package oraclecreator
+
+import (
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/services"
+
+	cctypes "github.com/smartcontractkit/chainlink/v2/core/capabilities/ccip/types"
+)
+
+// wrappedOracle is a wrapper for cctypes.CCIPOracle that allows custom actions on Oracle shutdown.
+type wrappedOracle struct {
+	baseOracle cctypes.CCIPOracle
+
+	// closableResources will be closed after calling baseOracle.Close()
+	closableResources []io.Closer
+}
+
+func newWrappedOracle(baseOracle cctypes.CCIPOracle, closableResources []io.Closer) cctypes.CCIPOracle {
+	return &wrappedOracle{
+		baseOracle:        baseOracle,
+		closableResources: closableResources,
+	}
+}
+
+func (o *wrappedOracle) Start() error {
+	return o.baseOracle.Start()
+}
+
+func (o *wrappedOracle) Close() error {
+	errs := make([]error, 0)
+
+	if err := o.baseOracle.Close(); err != nil {
+		errs = append(errs, fmt.Errorf("close base oracle: %w", err))
+	}
+
+	errs = append(errs, services.CloseAll(o.closableResources...))
+
+	return errors.Join(errs...)
+}

core/capabilities/ccip/oraclecreator/wrapped_oracle_test.go

@@ -0,0 +1,81 @@
+package oraclecreator
+
+import (
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_wrappedOracle_Close(t *testing.T) {
+	tests := []struct {
+		name         string
+		oracleErr    error
+		closerErrors []error
+		expectedErr  error
+	}{
+		{
+			name:        "no errors",
+			expectedErr: nil,
+		},
+		{
+			name:        "oracle error",
+			oracleErr:   err1,
+			expectedErr: errors.New("close base oracle: err1"),
+		},
+		{
+			name:         "oracle and closers errors",
+			oracleErr:    err1,
+			closerErrors: []error{nil, nil, err3},
+			expectedErr:  errors.New("close base oracle: err1\nerr3"),
+		},
+		{
+			name:         "closers only errors",
+			oracleErr:    nil,
+			closerErrors: []error{nil, err2, nil},
+			expectedErr:  err2,
+		},
+		{
+			name:         "no errors with closers",
+			closerErrors: []error{nil, nil, nil, nil},
+			expectedErr:  nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			closers := make([]io.Closer, 0, len(tt.closerErrors))
+			for _, err := range tt.closerErrors {
+				closers = append(closers, mockCloser{err: err})
+			}
+
+			o := newWrappedOracle(mockOracle{err: tt.oracleErr}, closers)
+
+			err := o.Close()
+			if err == nil && tt.expectedErr == nil {
+				assert.NoError(t, err)
+				return
+			}
+
+			assert.Error(t, err)
+			assert.Equal(t, tt.expectedErr.Error(), err.Error())
+		})
+	}
+}
+
+type mockCloser struct{ err error }
+
+func (m mockCloser) Close() error { return m.err }
+
+type mockOracle struct{ err error }
+
+func (m mockOracle) Close() error { return m.err }
+
+func (m mockOracle) Start() error { return m.err }
+
+var (
+	err1 = errors.New("err1")
+	err2 = errors.New("err2")
+	err3 = errors.New("err3")
+)