Merge pull request #529 from adamdecaf/extended-remarks

fix: copy extended remarks from sdn_comments.csv

cmd/server/download.go

@@ -174,6 +174,7 @@ func ofacRecords(logger log.Logger, initialDir string) (*ofac.Results, error) {
 			}
 			if rr != nil {
 				res = rr
+				res.SDNs = mergeSpilloverRecords(res.SDNs, res.SDNComments)
 			}
 		} else {
 			rr, err := ofac.Read(files[i])
@@ -184,13 +185,24 @@ func ofacRecords(logger log.Logger, initialDir string) (*ofac.Results, error) {
 				res.Addresses = append(res.Addresses, rr.Addresses...)
 				res.AlternateIdentities = append(res.AlternateIdentities, rr.AlternateIdentities...)
 				res.SDNs = append(res.SDNs, rr.SDNs...)
-				res.SDNComments = append(res.SDNComments, rr.SDNComments...)
+				res.SDNs = mergeSpilloverRecords(res.SDNs, rr.SDNComments)
 			}
 		}
 	}
 	return res, err
 }
 
+func mergeSpilloverRecords(sdns []*ofac.SDN, comments []*ofac.SDNComments) []*ofac.SDN {
+	for i := range sdns {
+		for j := range comments {
+			if sdns[i].EntityID == comments[j].EntityID {
+				sdns[i].Remarks += comments[j].RemarksExtended
+			}
+		}
+	}
+	return sdns
+}
+
 func dplRecords(logger log.Logger, initialDir string) ([]*dpl.DPL, error) {
 	file, err := dpl.Download(logger, initialDir)
 	if err != nil {

cmd/server/download_test.go

@@ -12,11 +12,13 @@ import (
 	"net/http/httptest"
 	"os"
 	"path/filepath"
+	"slices"
 	"testing"
 	"time"
 
 	"github.com/moov-io/base/log"
 	"github.com/moov-io/watchman/internal/database"
+	"github.com/moov-io/watchman/pkg/ofac"
 
 	"github.com/gorilla/mux"
 	"github.com/stretchr/testify/require"
@@ -206,3 +208,24 @@ func TestDownload__lastRefresh(t *testing.T) {
 		}
 	}
 }
+
+func TestDownload__OFAC_Spillover(t *testing.T) {
+	logger := log.NewTestLogger()
+	initialDir := filepath.Join("..", "..", "test", "testdata", "static")
+
+	res, err := ofacRecords(logger, initialDir)
+	require.NoError(t, err)
+
+	var sdn *ofac.SDN
+	idx := slices.IndexFunc(res.SDNs, func(s *ofac.SDN) bool {
+		return s.EntityID == "12300"
+	})
+	if idx >= 0 {
+		sdn = res.SDNs[idx]
+	}
+	require.NotNil(t, sdn)
+
+	//nolint:misspell
+	expected := `DOB 13 May 1965; alt. DOB 13 Apr 1968; alt. DOB 07 Jul 1964; POB Medellin, Colombia; alt. POB Marinilla, Antioquia, Colombia; alt. POB Ciudad Victoria, Tamaulipas, Mexico; Cedula No. 7548733 (Colombia); alt. Cedula No. 70163752 (Colombia); alt. Cedula No. 172489729-1 (Ecuador); Passport AL720622 (Colombia); R.F.C. CIVJ650513LJA (Mexico); alt. R.F.C. OUSV-640707 (Mexico); C.U.R.P. CIVJ650513HNEFLR06 (Mexico); alt. C.U.R.P. OUVS640707HTSSLR07 (Mexico); Matricula Mercantil No 181301-1 Cali (Colombia); alt. Matricula Mercantil No 405885 Bogota (Colombia); Linked To: BIO FORESTAL S.A.S.; Linked To: CUBI CAFE CLICK CUBE MEXICO, S.A. DE C.V.; Linked To: DOLPHIN DIVE SCHOOL S.A.; Linked To: GANADERIA LA SORGUITA S.A.S.; Linked To: GESTORES DEL ECUADOR GESTORUM S.A.; Linked To: INVERPUNTO DEL VALLE S.A.; Linked To: INVERSIONES CIFUENTES Y CIA. S. EN C.; Linked To: LE CLAUDE, S.A. DE C.V.; Linked To: OPERADORA NUEVA GRANADA, S.A. DE C.V.; Linked To: PARQUES TEMATICOS S.A.S.; Linked To: PROMO RAIZ S.A.S.; Linked To: RED MUNDIAL INMOBILIARIA, S.A. DE C.V.; Linked To: FUNDACION PARA EL BIENESTAR Y EL PORVENIR; Linked To: C.I. METALURGIA EXTRACTIVA DE COLOMBIA S.A.S.; Linked To: GRUPO MUNDO MARINO, S.A.; Linked To: C.I. DISERCOM S.A.S.; Linked To: C.I. OKCOFFEE COLOMBIA S.A.S.; Linked To: C.I. OKCOFFEE INTERNATIONAL S.A.S.; Linked To: FUNDACION OKCOFFEE COLOMBIA; Linked To: CUBICAFE S.A.S.; Linked To: HOTELES Y BIENES S.A.; Linked To: FUNDACION SALVA LA SELVA; Linked To: LINEA AEREA PUEBLOS AMAZONICOS S.A.S.; Linked To: DESARROLLO MINERO RESPONSABLE C.I. S.A.S.; Linked To: R D I S.A.`
+	require.Equal(t, expected, sdn.Remarks)
+}

pkg/ofac/reader_test.go

@@ -16,36 +16,32 @@ import (
 // TestOFAC__read validates reading an OFAC Address CSV File
 func TestOFAC__read(t *testing.T) {
 	res, err := Read(filepath.Join("..", "..", "test", "testdata", "add.csv"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(res.Addresses) == 0 {
-		t.Errorf("found no Addresses")
-	}
+	require.NoError(t, err)
+	require.Len(t, res.Addresses, 11696)
+	require.Len(t, res.AlternateIdentities, 0)
+	require.Len(t, res.SDNs, 0)
+	require.Len(t, res.SDNComments, 0)
 
 	res, err = Read(filepath.Join("..", "..", "test", "testdata", "alt.csv"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(res.AlternateIdentities) == 0 {
-		t.Errorf("found no AlternateIdentities")
-	}
+	require.NoError(t, err)
+	require.Len(t, res.Addresses, 0)
+	require.Len(t, res.AlternateIdentities, 9682)
+	require.Len(t, res.SDNs, 0)
+	require.Len(t, res.SDNComments, 0)
 
 	res, err = Read(filepath.Join("..", "..", "test", "testdata", "sdn.csv"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(res.SDNs) == 0 {
-		t.Errorf("found no SDNs")
-	}
+	require.NoError(t, err)
+	require.Len(t, res.Addresses, 0)
+	require.Len(t, res.AlternateIdentities, 0)
+	require.Len(t, res.SDNs, 7379)
+	require.Len(t, res.SDNComments, 0)
 
 	res, err = Read(filepath.Join("..", "..", "test", "testdata", "sdn_comments.csv"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(res.SDNComments) == 0 {
-		t.Errorf("found no SDN comments")
-	}
+	require.NoError(t, err)
+	require.Len(t, res.Addresses, 0)
+	require.Len(t, res.AlternateIdentities, 0)
+	require.Len(t, res.SDNs, 0)
+	require.Len(t, res.SDNComments, 13)
 }
 
 func TestReplaceNull(t *testing.T) {