From 4ffaae925ffc6f5e83cbb2a2a3963b3c3a703525 Mon Sep 17 00:00:00 2001 From: Neil O'Toole Date: Tue, 4 Jul 2023 11:31:47 -0600 Subject: [PATCH] #99: Rename duplicate ingest headers (#283) * CSV now renames duplicate ingest headers * Fix broken test * xlsx ingester now handles duplicate col names * Update CHANGELOG * Additional tests for ingest.column.rename * Removed dead comment in grammar --- CHANGELOG.md | 26 ++- .../upgrades/v0.34.0/upgrade_test.go | 7 +- cli/options.go | 6 +- cli/options_test.go | 2 +- cli/run.go | 4 +- drivers/csv/csv_test.go | 44 ++++ drivers/csv/detect_header.go | 9 +- drivers/csv/ingest.go | 8 +- drivers/csv/testdata/actor_duplicate_cols.csv | 201 ++++++++++++++++++ drivers/doc.go | 3 + drivers/drivers.go | 32 --- drivers/json/internal_test.go | 4 +- drivers/json/json.go | 4 +- drivers/mysql/metadata.go | 2 +- drivers/postgres/postgres.go | 2 +- drivers/sqlite3/metadata.go | 2 +- drivers/sqlserver/sqlserver.go | 2 +- drivers/xlsx/{import.go => ingest.go} | 13 +- .../xlsx/testdata/actor_duplicate_cols.xlsx | Bin 0 -> 16945 bytes drivers/xlsx/xlsx.go | 5 +- drivers/xlsx/xlsx_test.go | 52 ++++- grammar/SLQ.g4 | 2 +- libsq/driver/driver.go | 2 +- libsq/driver/driver_test.go | 2 +- libsq/driver/ingest.go | 95 +++++++++ libsq/driver/record.go | 15 +- testh/testh.go | 8 +- 27 files changed, 469 insertions(+), 83 deletions(-) create mode 100644 drivers/csv/testdata/actor_duplicate_cols.csv create mode 100644 drivers/doc.go delete mode 100644 drivers/drivers.go rename drivers/xlsx/{import.go => ingest.go} (97%) create mode 100644 drivers/xlsx/testdata/actor_duplicate_cols.xlsx create mode 100644 libsq/driver/ingest.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 542bb225a..9a3e61214 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Breaking changes are annotated with ☢️. +## Upcoming + +### Added + +- [#99]: The [CSV](https://sq.io/docs/drivers/csv) and [XLSX](https://sq.io/docs/drivers/xlsx) + drivers can now handle duplicate header column names. For example, given a CSV file: + + ```csv + actor_id,first_name,actor_id + 1,PENELOPE,1 + 2,NICK,2 + ``` + + The columns will be renamed to: + + ```csv + actor_id,first_name,actor_id_1 + ``` + + The renaming behavior is controlled by a new option `ingest.column.rename` + ([docs](https://sq.io/docs/config/#ingestcolumnrename)). + + ## [v0.40.0] - 2023-07-03 This release features a complete overhaul of the [`join`](https://sq.io/docs/query/#joins) @@ -18,7 +41,7 @@ mechanism. particularly useful, but it's a building block for [multiple joins](https://github.com/neilotoole/sq/issues/12). ```shell - $ sq `@sakila | .actor:a | .a.first_name` + $ sq '@sakila | .actor:a | .a.first_name' ``` - New option `result.column.rename` that exposes a template used to rename @@ -659,6 +682,7 @@ make working with lots of sources much easier. [#91]: https://github.com/neilotoole/sq/pull/91 [#95]: https://github.com/neilotoole/sq/issues/93 [#98]: https://github.com/neilotoole/sq/issues/98 +[#99]: https://github.com/neilotoole/sq/issues/99 [#123]: https://github.com/neilotoole/sq/issues/123 [#142]: https://github.com/neilotoole/sq/issues/142 [#144]: https://github.com/neilotoole/sq/issues/144 diff --git a/cli/config/yamlstore/upgrades/v0.34.0/upgrade_test.go b/cli/config/yamlstore/upgrades/v0.34.0/upgrade_test.go index beab91ab1..ae34f4210 100644 --- a/cli/config/yamlstore/upgrades/v0.34.0/upgrade_test.go +++ b/cli/config/yamlstore/upgrades/v0.34.0/upgrade_test.go @@ -8,10 +8,11 @@ import ( "testing" "time" + "github.com/neilotoole/sq/libsq/driver" + "github.com/neilotoole/sq/cli" "github.com/neilotoole/sq/cli/config/yamlstore" v0_34_0 "github.com/neilotoole/sq/cli/config/yamlstore/upgrades/v0.34.0" - "github.com/neilotoole/sq/drivers" "github.com/neilotoole/sq/drivers/csv" "github.com/neilotoole/sq/drivers/xlsx" "github.com/neilotoole/sq/libsq/core/options" @@ -80,12 +81,12 @@ func TestUpgrade(t *testing.T) { src1 := cfg.Collection.Sources()[1] require.Equal(t, handleCSV, src1.Handle) require.Equal(t, csv.TypeCSV, src1.Type) - require.Equal(t, true, src1.Options[drivers.OptIngestHeader.Key()]) + require.Equal(t, true, src1.Options[driver.OptIngestHeader.Key()]) src2 := cfg.Collection.Sources()[2] require.Equal(t, handleXLSX, src2.Handle) require.Equal(t, xlsx.Type, src2.Type) - require.Equal(t, false, src2.Options[drivers.OptIngestHeader.Key()]) + require.Equal(t, false, src2.Options[driver.OptIngestHeader.Key()]) wantCfgRaw, err := os.ReadFile(filepath.Join("testdata", "want.sq.yml")) require.NoError(t, err) diff --git a/cli/options.go b/cli/options.go index 9e00a41ef..5ede66115 100644 --- a/cli/options.go +++ b/cli/options.go @@ -8,7 +8,6 @@ import ( "github.com/neilotoole/sq/libsq/core/timez" - "github.com/neilotoole/sq/drivers" "github.com/neilotoole/sq/drivers/csv" "github.com/neilotoole/sq/libsq/core/errz" "github.com/neilotoole/sq/libsq/core/options" @@ -162,8 +161,9 @@ func RegisterDefaultOpts(reg *options.Registry) { driver.OptTuningErrgroupLimit, driver.OptTuningRecChanSize, OptTuningFlushThreshold, - drivers.OptIngestHeader, - drivers.OptIngestSampleSize, + driver.OptIngestHeader, + driver.OptIngestColRename, + driver.OptIngestSampleSize, csv.OptDelim, csv.OptEmptyAsNull, ) diff --git a/cli/options_test.go b/cli/options_test.go index 21d897dad..dd8559a37 100644 --- a/cli/options_test.go +++ b/cli/options_test.go @@ -19,7 +19,7 @@ func TestRegisterDefaultOpts(t *testing.T) { log.Debug("options.Registry (after)", "reg", reg) keys := reg.Keys() - require.Len(t, keys, 32) + require.Len(t, keys, 33) for _, opt := range reg.Opts() { opt := opt diff --git a/cli/run.go b/cli/run.go index 7c8ef5ada..e430e68c4 100644 --- a/cli/run.go +++ b/cli/run.go @@ -8,8 +8,6 @@ import ( "github.com/neilotoole/sq/cli/run" - "github.com/neilotoole/sq/drivers" - "github.com/neilotoole/sq/cli/config/yamlstore" v0_34_0 "github.com/neilotoole/sq/cli/config/yamlstore/upgrades/v0.34.0" "github.com/neilotoole/sq/libsq/core/lg/slogbuf" @@ -175,7 +173,7 @@ func FinishRunInit(ctx context.Context, ru *run.Run) error { dr.AddProvider(json.TypeJSON, jsonp) dr.AddProvider(json.TypeJSONA, jsonp) dr.AddProvider(json.TypeJSONL, jsonp) - sampleSize := drivers.OptIngestSampleSize.Get(cfg.Options) + sampleSize := driver.OptIngestSampleSize.Get(cfg.Options) ru.Files.AddDriverDetectors( json.DetectJSON(sampleSize), json.DetectJSONA(sampleSize), diff --git a/drivers/csv/csv_test.go b/drivers/csv/csv_test.go index f630d98dc..b55c1cb46 100644 --- a/drivers/csv/csv_test.go +++ b/drivers/csv/csv_test.go @@ -1,8 +1,14 @@ package csv_test import ( + "context" + "path/filepath" "testing" + "github.com/neilotoole/sq/libsq/driver" + + "github.com/neilotoole/sq/cli/testrun" + "github.com/neilotoole/sq/libsq/core/timez" "github.com/neilotoole/sq/libsq/core/stringz" @@ -88,3 +94,41 @@ func TestEmptyAsNull(t *testing.T) { require.EqualValues(t, want[i], rec0[i], "field [%d]", i) } } + +func TestIngestDuplicateColumns(t *testing.T) { + ctx := context.Background() + tr := testrun.New(ctx, t, nil) + + err := tr.Exec( + "add", filepath.Join("testdata", "actor_duplicate_cols.csv"), + "--handle", "@actor_dup", + ) + require.NoError(t, err) + + tr = testrun.New(ctx, t, tr).Hush() + require.NoError(t, tr.Exec("--csv", ".data")) + wantHeaders := []string{"actor_id", "first_name", "last_name", "last_update", "actor_id_1"} + data := tr.MustReadCSV() + require.Equal(t, wantHeaders, data[0]) + + // Make sure the data is correct + require.Len(t, data, sakila.TblActorCount+1) // +1 for header row + wantFirstDataRecord := []string{"1", "PENELOPE", "GUINESS", "2020-02-15T06:59:28Z", "1"} + require.Equal(t, wantFirstDataRecord, data[1]) + + // Verify that changing the template works + const tpl2 = "x_{{.Name}}{{with .Recurrence}}_{{.}}{{end}}" + + tr = testrun.New(ctx, t, tr) + require.NoError(t, tr.Exec( + "config", + "set", + driver.OptIngestColRename.Key(), + tpl2, + )) + tr = testrun.New(ctx, t, tr) + require.NoError(t, tr.Exec("--csv", ".data")) + wantHeaders = []string{"x_actor_id", "x_first_name", "x_last_name", "x_last_update", "x_actor_id_1"} + data = tr.MustReadCSV() + require.Equal(t, wantHeaders, data[0]) +} diff --git a/drivers/csv/detect_header.go b/drivers/csv/detect_header.go index 9b6152d35..eca697e0c 100644 --- a/drivers/csv/detect_header.go +++ b/drivers/csv/detect_header.go @@ -4,10 +4,11 @@ import ( "context" "strings" + "github.com/neilotoole/sq/libsq/driver" + "github.com/neilotoole/sq/libsq/core/lg" "github.com/neilotoole/sq/libsq/core/lg/lga" - "github.com/neilotoole/sq/drivers" "github.com/neilotoole/sq/libsq/core/options" "github.com/neilotoole/sq/libsq/core/errz" @@ -18,10 +19,10 @@ import ( // set in opts, or if detectHeaderRow detects that the first // row of recs seems to be a header. func hasHeaderRow(ctx context.Context, recs [][]string, opts options.Options) (bool, error) { - if drivers.OptIngestHeader.IsSet(opts) { - b := drivers.OptIngestHeader.Get(opts) + if driver.OptIngestHeader.IsSet(opts) { + b := driver.OptIngestHeader.Get(opts) lg.FromContext(ctx).Debug("CSV ingest header explicitly specified: skipping header detection", - lga.Key, drivers.OptIngestHeader.Key(), + lga.Key, driver.OptIngestHeader.Key(), lga.Val, b) return b, nil } diff --git a/drivers/csv/ingest.go b/drivers/csv/ingest.go index e350110a3..16090e8b0 100644 --- a/drivers/csv/ingest.go +++ b/drivers/csv/ingest.go @@ -9,8 +9,6 @@ import ( "github.com/neilotoole/sq/libsq/core/record" - "github.com/neilotoole/sq/drivers" - "github.com/neilotoole/sq/libsq/core/kind" "github.com/neilotoole/sq/libsq/core/stringz" @@ -74,7 +72,7 @@ func ingestCSV(ctx context.Context, src *source.Source, openFn source.FileOpenFu } cr := newCSVReader(r, delim) - recs, err := readRecords(cr, drivers.OptIngestSampleSize.Get(src.Options)) + recs, err := readRecords(cr, driver.OptIngestSampleSize.Get(src.Options)) if err != nil { return err } @@ -99,6 +97,10 @@ func ingestCSV(ctx context.Context, src *source.Source, openFn source.FileOpenFu } } + if header, err = driver.MungeIngestColNames(ctx, header); err != nil { + return err + } + kinds, mungers, err := detectColKinds(recs) if err != nil { return err diff --git a/drivers/csv/testdata/actor_duplicate_cols.csv b/drivers/csv/testdata/actor_duplicate_cols.csv new file mode 100644 index 000000000..f0caebb29 --- /dev/null +++ b/drivers/csv/testdata/actor_duplicate_cols.csv @@ -0,0 +1,201 @@ +actor_id,first_name,last_name,last_update,actor_id +1,PENELOPE,GUINESS,2020-02-15T06:59:28Z,1 +2,NICK,WAHLBERG,2020-02-15T06:59:28Z,2 +3,ED,CHASE,2020-02-15T06:59:28Z,3 +4,JENNIFER,DAVIS,2020-02-15T06:59:28Z,4 +5,JOHNNY,LOLLOBRIGIDA,2020-02-15T06:59:28Z,5 +6,BETTE,NICHOLSON,2020-02-15T06:59:28Z,6 +7,GRACE,MOSTEL,2020-02-15T06:59:28Z,7 +8,MATTHEW,JOHANSSON,2020-02-15T06:59:28Z,8 +9,JOE,SWANK,2020-02-15T06:59:28Z,9 +10,CHRISTIAN,GABLE,2020-02-15T06:59:28Z,10 +11,ZERO,CAGE,2020-02-15T06:59:28Z,11 +12,KARL,BERRY,2020-02-15T06:59:28Z,12 +13,UMA,WOOD,2020-02-15T06:59:28Z,13 +14,VIVIEN,BERGEN,2020-02-15T06:59:28Z,14 +15,CUBA,OLIVIER,2020-02-15T06:59:28Z,15 +16,FRED,COSTNER,2020-02-15T06:59:28Z,16 +17,HELEN,VOIGHT,2020-02-15T06:59:28Z,17 +18,DAN,TORN,2020-02-15T06:59:28Z,18 +19,BOB,FAWCETT,2020-02-15T06:59:28Z,19 +20,LUCILLE,TRACY,2020-02-15T06:59:28Z,20 +21,KIRSTEN,PALTROW,2020-02-15T06:59:28Z,21 +22,ELVIS,MARX,2020-02-15T06:59:28Z,22 +23,SANDRA,KILMER,2020-02-15T06:59:28Z,23 +24,CAMERON,STREEP,2020-02-15T06:59:28Z,24 +25,KEVIN,BLOOM,2020-02-15T06:59:28Z,25 +26,RIP,CRAWFORD,2020-02-15T06:59:28Z,26 +27,JULIA,MCQUEEN,2020-02-15T06:59:28Z,27 +28,WOODY,HOFFMAN,2020-02-15T06:59:28Z,28 +29,ALEC,WAYNE,2020-02-15T06:59:28Z,29 +30,SANDRA,PECK,2020-02-15T06:59:28Z,30 +31,SISSY,SOBIESKI,2020-02-15T06:59:28Z,31 +32,TIM,HACKMAN,2020-02-15T06:59:28Z,32 +33,MILLA,PECK,2020-02-15T06:59:28Z,33 +34,AUDREY,OLIVIER,2020-02-15T06:59:28Z,34 +35,JUDY,DEAN,2020-02-15T06:59:28Z,35 +36,BURT,DUKAKIS,2020-02-15T06:59:28Z,36 +37,VAL,BOLGER,2020-02-15T06:59:28Z,37 +38,TOM,MCKELLEN,2020-02-15T06:59:28Z,38 +39,GOLDIE,BRODY,2020-02-15T06:59:28Z,39 +40,JOHNNY,CAGE,2020-02-15T06:59:28Z,40 +41,JODIE,DEGENERES,2020-02-15T06:59:28Z,41 +42,TOM,MIRANDA,2020-02-15T06:59:28Z,42 +43,KIRK,JOVOVICH,2020-02-15T06:59:28Z,43 +44,NICK,STALLONE,2020-02-15T06:59:28Z,44 +45,REESE,KILMER,2020-02-15T06:59:28Z,45 +46,PARKER,GOLDBERG,2020-02-15T06:59:28Z,46 +47,JULIA,BARRYMORE,2020-02-15T06:59:28Z,47 +48,FRANCES,DAY-LEWIS,2020-02-15T06:59:28Z,48 +49,ANNE,CRONYN,2020-02-15T06:59:28Z,49 +50,NATALIE,HOPKINS,2020-02-15T06:59:28Z,50 +51,GARY,PHOENIX,2020-02-15T06:59:28Z,51 +52,CARMEN,HUNT,2020-02-15T06:59:28Z,52 +53,MENA,TEMPLE,2020-02-15T06:59:28Z,53 +54,PENELOPE,PINKETT,2020-02-15T06:59:28Z,54 +55,FAY,KILMER,2020-02-15T06:59:28Z,55 +56,DAN,HARRIS,2020-02-15T06:59:28Z,56 +57,JUDE,CRUISE,2020-02-15T06:59:28Z,57 +58,CHRISTIAN,AKROYD,2020-02-15T06:59:28Z,58 +59,DUSTIN,TAUTOU,2020-02-15T06:59:28Z,59 +60,HENRY,BERRY,2020-02-15T06:59:28Z,60 +61,CHRISTIAN,NEESON,2020-02-15T06:59:28Z,61 +62,JAYNE,NEESON,2020-02-15T06:59:28Z,62 +63,CAMERON,WRAY,2020-02-15T06:59:28Z,63 +64,RAY,JOHANSSON,2020-02-15T06:59:28Z,64 +65,ANGELA,HUDSON,2020-02-15T06:59:28Z,65 +66,MARY,TANDY,2020-02-15T06:59:28Z,66 +67,JESSICA,BAILEY,2020-02-15T06:59:28Z,67 +68,RIP,WINSLET,2020-02-15T06:59:28Z,68 +69,KENNETH,PALTROW,2020-02-15T06:59:28Z,69 +70,MICHELLE,MCCONAUGHEY,2020-02-15T06:59:28Z,70 +71,ADAM,GRANT,2020-02-15T06:59:28Z,71 +72,SEAN,WILLIAMS,2020-02-15T06:59:28Z,72 +73,GARY,PENN,2020-02-15T06:59:28Z,73 +74,MILLA,KEITEL,2020-02-15T06:59:28Z,74 +75,BURT,POSEY,2020-02-15T06:59:28Z,75 +76,ANGELINA,ASTAIRE,2020-02-15T06:59:28Z,76 +77,CARY,MCCONAUGHEY,2020-02-15T06:59:28Z,77 +78,GROUCHO,SINATRA,2020-02-15T06:59:28Z,78 +79,MAE,HOFFMAN,2020-02-15T06:59:28Z,79 +80,RALPH,CRUZ,2020-02-15T06:59:28Z,80 +81,SCARLETT,DAMON,2020-02-15T06:59:28Z,81 +82,WOODY,JOLIE,2020-02-15T06:59:28Z,82 +83,BEN,WILLIS,2020-02-15T06:59:28Z,83 +84,JAMES,PITT,2020-02-15T06:59:28Z,84 +85,MINNIE,ZELLWEGER,2020-02-15T06:59:28Z,85 +86,GREG,CHAPLIN,2020-02-15T06:59:28Z,86 +87,SPENCER,PECK,2020-02-15T06:59:28Z,87 +88,KENNETH,PESCI,2020-02-15T06:59:28Z,88 +89,CHARLIZE,DENCH,2020-02-15T06:59:28Z,89 +90,SEAN,GUINESS,2020-02-15T06:59:28Z,90 +91,CHRISTOPHER,BERRY,2020-02-15T06:59:28Z,91 +92,KIRSTEN,AKROYD,2020-02-15T06:59:28Z,92 +93,ELLEN,PRESLEY,2020-02-15T06:59:28Z,93 +94,KENNETH,TORN,2020-02-15T06:59:28Z,94 +95,DARYL,WAHLBERG,2020-02-15T06:59:28Z,95 +96,GENE,WILLIS,2020-02-15T06:59:28Z,96 +97,MEG,HAWKE,2020-02-15T06:59:28Z,97 +98,CHRIS,BRIDGES,2020-02-15T06:59:28Z,98 +99,JIM,MOSTEL,2020-02-15T06:59:28Z,99 +100,SPENCER,DEPP,2020-02-15T06:59:28Z,100 +101,SUSAN,DAVIS,2020-02-15T06:59:28Z,101 +102,WALTER,TORN,2020-02-15T06:59:28Z,102 +103,MATTHEW,LEIGH,2020-02-15T06:59:28Z,103 +104,PENELOPE,CRONYN,2020-02-15T06:59:28Z,104 +105,SIDNEY,CROWE,2020-02-15T06:59:28Z,105 +106,GROUCHO,DUNST,2020-02-15T06:59:28Z,106 +107,GINA,DEGENERES,2020-02-15T06:59:28Z,107 +108,WARREN,NOLTE,2020-02-15T06:59:28Z,108 +109,SYLVESTER,DERN,2020-02-15T06:59:28Z,109 +110,SUSAN,DAVIS,2020-02-15T06:59:28Z,110 +111,CAMERON,ZELLWEGER,2020-02-15T06:59:28Z,111 +112,RUSSELL,BACALL,2020-02-15T06:59:28Z,112 +113,MORGAN,HOPKINS,2020-02-15T06:59:28Z,113 +114,MORGAN,MCDORMAND,2020-02-15T06:59:28Z,114 +115,HARRISON,BALE,2020-02-15T06:59:28Z,115 +116,DAN,STREEP,2020-02-15T06:59:28Z,116 +117,RENEE,TRACY,2020-02-15T06:59:28Z,117 +118,CUBA,ALLEN,2020-02-15T06:59:28Z,118 +119,WARREN,JACKMAN,2020-02-15T06:59:28Z,119 +120,PENELOPE,MONROE,2020-02-15T06:59:28Z,120 +121,LIZA,BERGMAN,2020-02-15T06:59:28Z,121 +122,SALMA,NOLTE,2020-02-15T06:59:28Z,122 +123,JULIANNE,DENCH,2020-02-15T06:59:28Z,123 +124,SCARLETT,BENING,2020-02-15T06:59:28Z,124 +125,ALBERT,NOLTE,2020-02-15T06:59:28Z,125 +126,FRANCES,TOMEI,2020-02-15T06:59:28Z,126 +127,KEVIN,GARLAND,2020-02-15T06:59:28Z,127 +128,CATE,MCQUEEN,2020-02-15T06:59:28Z,128 +129,DARYL,CRAWFORD,2020-02-15T06:59:28Z,129 +130,GRETA,KEITEL,2020-02-15T06:59:28Z,130 +131,JANE,JACKMAN,2020-02-15T06:59:28Z,131 +132,ADAM,HOPPER,2020-02-15T06:59:28Z,132 +133,RICHARD,PENN,2020-02-15T06:59:28Z,133 +134,GENE,HOPKINS,2020-02-15T06:59:28Z,134 +135,RITA,REYNOLDS,2020-02-15T06:59:28Z,135 +136,ED,MANSFIELD,2020-02-15T06:59:28Z,136 +137,MORGAN,WILLIAMS,2020-02-15T06:59:28Z,137 +138,LUCILLE,DEE,2020-02-15T06:59:28Z,138 +139,EWAN,GOODING,2020-02-15T06:59:28Z,139 +140,WHOOPI,HURT,2020-02-15T06:59:28Z,140 +141,CATE,HARRIS,2020-02-15T06:59:28Z,141 +142,JADA,RYDER,2020-02-15T06:59:28Z,142 +143,RIVER,DEAN,2020-02-15T06:59:28Z,143 +144,ANGELA,WITHERSPOON,2020-02-15T06:59:28Z,144 +145,KIM,ALLEN,2020-02-15T06:59:28Z,145 +146,ALBERT,JOHANSSON,2020-02-15T06:59:28Z,146 +147,FAY,WINSLET,2020-02-15T06:59:28Z,147 +148,EMILY,DEE,2020-02-15T06:59:28Z,148 +149,RUSSELL,TEMPLE,2020-02-15T06:59:28Z,149 +150,JAYNE,NOLTE,2020-02-15T06:59:28Z,150 +151,GEOFFREY,HESTON,2020-02-15T06:59:28Z,151 +152,BEN,HARRIS,2020-02-15T06:59:28Z,152 +153,MINNIE,KILMER,2020-02-15T06:59:28Z,153 +154,MERYL,GIBSON,2020-02-15T06:59:28Z,154 +155,IAN,TANDY,2020-02-15T06:59:28Z,155 +156,FAY,WOOD,2020-02-15T06:59:28Z,156 +157,GRETA,MALDEN,2020-02-15T06:59:28Z,157 +158,VIVIEN,BASINGER,2020-02-15T06:59:28Z,158 +159,LAURA,BRODY,2020-02-15T06:59:28Z,159 +160,CHRIS,DEPP,2020-02-15T06:59:28Z,160 +161,HARVEY,HOPE,2020-02-15T06:59:28Z,161 +162,OPRAH,KILMER,2020-02-15T06:59:28Z,162 +163,CHRISTOPHER,WEST,2020-02-15T06:59:28Z,163 +164,HUMPHREY,WILLIS,2020-02-15T06:59:28Z,164 +165,AL,GARLAND,2020-02-15T06:59:28Z,165 +166,NICK,DEGENERES,2020-02-15T06:59:28Z,166 +167,LAURENCE,BULLOCK,2020-02-15T06:59:28Z,167 +168,WILL,WILSON,2020-02-15T06:59:28Z,168 +169,KENNETH,HOFFMAN,2020-02-15T06:59:28Z,169 +170,MENA,HOPPER,2020-02-15T06:59:28Z,170 +171,OLYMPIA,PFEIFFER,2020-02-15T06:59:28Z,171 +172,GROUCHO,WILLIAMS,2020-02-15T06:59:28Z,172 +173,ALAN,DREYFUSS,2020-02-15T06:59:28Z,173 +174,MICHAEL,BENING,2020-02-15T06:59:28Z,174 +175,WILLIAM,HACKMAN,2020-02-15T06:59:28Z,175 +176,JON,CHASE,2020-02-15T06:59:28Z,176 +177,GENE,MCKELLEN,2020-02-15T06:59:28Z,177 +178,LISA,MONROE,2020-02-15T06:59:28Z,178 +179,ED,GUINESS,2020-02-15T06:59:28Z,179 +180,JEFF,SILVERSTONE,2020-02-15T06:59:28Z,180 +181,MATTHEW,CARREY,2020-02-15T06:59:28Z,181 +182,DEBBIE,AKROYD,2020-02-15T06:59:28Z,182 +183,RUSSELL,CLOSE,2020-02-15T06:59:28Z,183 +184,HUMPHREY,GARLAND,2020-02-15T06:59:28Z,184 +185,MICHAEL,BOLGER,2020-02-15T06:59:28Z,185 +186,JULIA,ZELLWEGER,2020-02-15T06:59:28Z,186 +187,RENEE,BALL,2020-02-15T06:59:28Z,187 +188,ROCK,DUKAKIS,2020-02-15T06:59:28Z,188 +189,CUBA,BIRCH,2020-02-15T06:59:28Z,189 +190,AUDREY,BAILEY,2020-02-15T06:59:28Z,190 +191,GREGORY,GOODING,2020-02-15T06:59:28Z,191 +192,JOHN,SUVARI,2020-02-15T06:59:28Z,192 +193,BURT,TEMPLE,2020-02-15T06:59:28Z,193 +194,MERYL,ALLEN,2020-02-15T06:59:28Z,194 +195,JAYNE,SILVERSTONE,2020-02-15T06:59:28Z,195 +196,BELA,WALKEN,2020-02-15T06:59:28Z,196 +197,REESE,WEST,2020-02-15T06:59:28Z,197 +198,MARY,KEITEL,2020-02-15T06:59:28Z,198 +199,JULIA,FAWCETT,2020-02-15T06:59:28Z,199 +200,THORA,TEMPLE,2020-02-15T06:59:28Z,200 diff --git a/drivers/doc.go b/drivers/doc.go new file mode 100644 index 000000000..81e237767 --- /dev/null +++ b/drivers/doc.go @@ -0,0 +1,3 @@ +// Package drivers is the parent package of the +// concrete sq driver implementations. +package drivers diff --git a/drivers/drivers.go b/drivers/drivers.go deleted file mode 100644 index a7c7987ab..000000000 --- a/drivers/drivers.go +++ /dev/null @@ -1,32 +0,0 @@ -// Package drivers is the parent package of the -// concrete sq driver implementations. -package drivers - -import "github.com/neilotoole/sq/libsq/core/options" - -// OptIngestHeader specifies whether ingested data has a header row or not. -// If not set, the ingester *may* try to detect if the input has a header. -var OptIngestHeader = options.NewBool( - "ingest.header", - "", - 0, - false, - "Ingest data has a header row", - `Specifies whether ingested data has a header row or not. -If not set, the ingester *may* try to detect if the input has a header. -Generally it is best to leave this option unset and allow the ingester -to detect the header.`, - "source", -) - -// OptIngestSampleSize specifies the number of samples that a detector -// should take to determine type. -var OptIngestSampleSize = options.NewInt( - "ingest.sample-size", - "", - 0, - 1024, - "Ingest data sample size for type detection", - `Specify the number of samples that a detector should take to determine type.`, - "source", -) diff --git a/drivers/json/internal_test.go b/drivers/json/internal_test.go index 0a36edccf..eea4edafc 100644 --- a/drivers/json/internal_test.go +++ b/drivers/json/internal_test.go @@ -7,8 +7,6 @@ import ( "os" "testing" - "github.com/neilotoole/sq/drivers" - "github.com/stretchr/testify/require" "github.com/neilotoole/sq/libsq/core/kind" @@ -32,7 +30,7 @@ func newImportJob(fromSrc *source.Source, openFn source.FileOpenFunc, destDB dri flatten bool, ) importJob { if sampleSize <= 0 { - sampleSize = drivers.OptIngestSampleSize.Get(fromSrc.Options) + sampleSize = driver.OptIngestSampleSize.Get(fromSrc.Options) } return importJob{ diff --git a/drivers/json/json.go b/drivers/json/json.go index 359e49331..e25458ac3 100644 --- a/drivers/json/json.go +++ b/drivers/json/json.go @@ -9,8 +9,6 @@ import ( "context" "database/sql" - "github.com/neilotoole/sq/drivers" - "github.com/neilotoole/sq/libsq/core/lg/lga" "github.com/neilotoole/sq/libsq/core/lg/lgm" @@ -117,7 +115,7 @@ func (d *driveri) Open(ctx context.Context, src *source.Source) (driver.Database fromSrc: src, openFn: d.files.OpenFunc(src), destDB: dbase.impl, - sampleSize: drivers.OptIngestSampleSize.Get(src.Options), + sampleSize: driver.OptIngestSampleSize.Get(src.Options), flatten: true, } diff --git a/drivers/mysql/metadata.go b/drivers/mysql/metadata.go index aa1872171..998c0cf80 100644 --- a/drivers/mysql/metadata.go +++ b/drivers/mysql/metadata.go @@ -99,7 +99,7 @@ func recordMetaFromColumnTypes(ctx context.Context, colTypes []*sql.ColumnType) ogColNames[i] = colTypeData.Name } - mungedColNames, err := driver.MungeColNames(ctx, ogColNames) + mungedColNames, err := driver.MungeResultColNames(ctx, ogColNames) if err != nil { return nil, err } diff --git a/drivers/postgres/postgres.go b/drivers/postgres/postgres.go index 543dbdcb2..f325f02e5 100644 --- a/drivers/postgres/postgres.go +++ b/drivers/postgres/postgres.go @@ -540,7 +540,7 @@ func (d *driveri) RecordMeta(ctx context.Context, colTypes []*sql.ColumnType) (r ogColNames[i] = colTypeData.Name } - mungedColNames, err := driver.MungeColNames(ctx, ogColNames) + mungedColNames, err := driver.MungeResultColNames(ctx, ogColNames) if err != nil { return nil, nil, err } diff --git a/drivers/sqlite3/metadata.go b/drivers/sqlite3/metadata.go index c5164305b..5e811152e 100644 --- a/drivers/sqlite3/metadata.go +++ b/drivers/sqlite3/metadata.go @@ -45,7 +45,7 @@ func recordMetaFromColumnTypes(ctx context.Context, colTypes []*sql.ColumnType) ogColNames[i] = colTypeData.Name } - mungedColNames, err := driver.MungeColNames(ctx, ogColNames) + mungedColNames, err := driver.MungeResultColNames(ctx, ogColNames) if err != nil { return nil, err } diff --git a/drivers/sqlserver/sqlserver.go b/drivers/sqlserver/sqlserver.go index 5f2f60ae4..19cb62dd4 100644 --- a/drivers/sqlserver/sqlserver.go +++ b/drivers/sqlserver/sqlserver.go @@ -302,7 +302,7 @@ func (d *driveri) RecordMeta(ctx context.Context, colTypes []*sql.ColumnType) (r ogColNames[i] = colTypeData.Name } - mungedColNames, err := driver.MungeColNames(ctx, ogColNames) + mungedColNames, err := driver.MungeResultColNames(ctx, ogColNames) if err != nil { return nil, nil, err } diff --git a/drivers/xlsx/import.go b/drivers/xlsx/ingest.go similarity index 97% rename from drivers/xlsx/import.go rename to drivers/xlsx/ingest.go index de32c1244..7f0823888 100644 --- a/drivers/xlsx/import.go +++ b/drivers/xlsx/ingest.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/neilotoole/sq/drivers" "github.com/neilotoole/sq/libsq/core/lg/lga" "github.com/neilotoole/sq/libsq/core/lg/lgm" @@ -35,7 +34,7 @@ func xlsxToScratch(ctx context.Context, src *source.Source, xlFile *xlsx.File, s lga.Src, src, lga.Target, scratchDB.Source()) - hasHeader := drivers.OptIngestHeader.Get(src.Options) + hasHeader := driver.OptIngestHeader.Get(src.Options) // TODO: Like the csv driver, the xlsx driver should detect // the presence of a header. @@ -189,7 +188,7 @@ func buildTblDefsForSheets(ctx context.Context, sheets []*xlsx.Sheet, hasHeader default: } - tblDef, err := buildTblDefForSheet(lg.FromContext(gCtx), sheets[i], hasHeader) + tblDef, err := buildTblDefForSheet(gCtx, sheets[i], hasHeader) if err != nil { return err } @@ -208,7 +207,8 @@ func buildTblDefsForSheets(ctx context.Context, sheets []*xlsx.Sheet, hasHeader // buildTblDefForSheet creates a table for the given sheet, and returns // a model of the table, or an error. If the sheet is empty, (nil,nil) // is returned. -func buildTblDefForSheet(log *slog.Logger, sheet *xlsx.Sheet, hasHeader bool) (*sqlmodel.TableDef, error) { +func buildTblDefForSheet(ctx context.Context, sheet *xlsx.Sheet, hasHeader bool) (*sqlmodel.TableDef, error) { + log := lg.FromContext(ctx) maxCols := getRowsMaxCellCount(sheet) if maxCols == 0 { log.Warn("sheet is empty: skipping", "sheet", sheet.Name) @@ -263,6 +263,11 @@ func buildTblDefForSheet(log *slog.Logger, sheet *xlsx.Sheet, hasHeader bool) (* colNames, colKinds = syncColNamesKinds(colNames, colKinds) + var err error + if colNames, err = driver.MungeIngestColNames(ctx, colNames); err != nil { + return nil, err + } + tblDef := &sqlmodel.TableDef{Name: sheet.Name} cols := make([]*sqlmodel.ColDef, len(colNames)) for i, colName := range colNames { diff --git a/drivers/xlsx/testdata/actor_duplicate_cols.xlsx b/drivers/xlsx/testdata/actor_duplicate_cols.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..88043600bb53b2c982766261789e9c07d221b681 GIT binary patch literal 16945 zcmeHu1y@|#vTkF+f?IG1?h+h=2M7cR?(Xic!4urwAz0(??he7--96ZAlD*$PC+EEP z2kz}Lnx@C>syS!X_f^$eYu0B82uO6mD*y}t03Zge?5@apf&l>8PyhfL00vx5$jZ{* zz|vkz-o@I$PJ`ar!kj1z5}Yyv01o>7|GxeQTL2g{DAmb`{NYCIL1>fCbg_aThV{s& z7nNFuw=I++RH2uRpq9mJw9A7S9VgQet+#w*xiY;4;PF2*WZWv#FLa;=+E zn>6K+p4@oLXNySy^4Egd?c^I2lwGQUn=TURPNW1Z6`MA-1|4kZ-qeNxbS` zUn5d|=y|V99mww2?p!2Pd0yMjdmLKQLj&9Y>GqVGx?@#sxD}{H2h_;j`JhT^A(M}n zGdzQ_#Q$x?H$oZQPdv}&6HVsOm-`T}#Sk9E>@j{;F6`zrwlDmWoJ-o1$0BR+lsu||sDSQQU!MR#&9CimSd0O0u<0`U3Y zXj`qoNOB3n;wKO*;X$<3vNbTbW1xTe`@eMkAI#Bz**Y*vQo57zb-;<(Ltx*{)KUbp zpoEivSR=8#r}xK2r0URYQrx9h3Orva}jFe*AXNsUu+ zK>SY|2WV<%nQ%GsD7_L6ELtvJ z0OtF|FRDF~D$BaJC16v6a!17hWwq=X`_ZGxUNdpu_YnAkI6oauBx3a0=^D+Ixc8Y8 zU)|v-$Qg5*l9r5KeAIdvvFhO6ik*%Yu5#JSy5tmT~2!TE%iUnCjBS z4%=DOciQh0Oqjn6%3%LV5)q6R3UJVf5Ci~#1AqZ@HfQ+F6DKQMGhHhyvzP4lH)p^= zDGv18|Jg5K)UYWi#QUB2xA@Mw(Q!X!`t{Yv2$op<1PE%Xk=N)~EfBVP_-EL>Zg0MQ zo1npRQ9fk(Jn6I!4^h2LMOlFd$+b~5%?igGR_ZiL6jWVgzk^TxCQ@5#dHHa;Y!Z(z z(gGTFygL$nkjFfiRIhv{<_=6_@dxi^QugamyeiJUS6k$x>H2k--nQFRh_OD_`Nsly zy~O0tCr1YjthejKM&V(l!eSe#LeXW}#fS(MKEXPeCIXt!b3oXRGky+YQmRQcS|)Te zX7_vRx50gAy)MGc7MMB&Gk(NH5Q-;wx@X4VNulxd7iHA1!!-`?>WO16wVF~CS|335 z&wnC_PH47!5ENp4K|vS^1Pu^^{uMhp3Nq3QjL4qpRnHMB9tdHRY;-bmkdRg43-ayW zqMPKFdTX=oj>ahKttTmLxw#Clvwn=+ZH}%idF+HqDdwK0C9tXYLq=u;Ts|36@7P9W zB)7&tA-nX*Cdy@!N{qdGx2yl~xL{mx+4-pe8VQi|@w1KvhMswaPwWr8f^Q#Bhhpgq zpyU|J3Rtr1soqrPw&@@X*X5{Zu?hnDd@{G~(#qux9Qt!Ke2vNTp?-c}jWrn`I3%$q zn?iEfGAaZDbbFv`M-eE`2WZo1b0+M`i4HOL+-MN2Ao)<+kZUNu9IWO?8L48k)`|i` zXL69%DZt3}8K8Z>493@x`ml_ zW;_Dbs39?vt#B0>S0W4U;)CNBX*YNdiO?o5QB#|7un*h@ZzAPD3_~HX+Txkg=0B45 zH&B%w9Eq(Wsz$gf9Vl$)Wk3&_ZPX2Coq6pFO$xczCRUj+G88l4QB*C4+`@!R7#9L> z5R3Y{v8Ho-yt#h6#O6#q0iy<4M~beSyCNs>{V8EbPU0YZfI9H5q;+Kn%J`rTW95e_ zQVYc%17#blhmS4)_=RMxlzL0!4f7;Zz1PR=WABKn10)9D7O%+bh9{Ny*AHbY(xYyj z7pvy{n#6DLyA!jj>DXc&$6)sbRglzM_oxM$TP`AqN?e{qHppgqT9Q`~ichAX4v(wU zM&h<5(MB#6oz%BYVToS{5DQ-|&aV-H+s!$jI9tMNV_gR>RHuh9z-8b$3e_6#9S&$H zl_6i^O^q>0juCX)51*nEtRT3(K11x@W`immKJ5PvVQd&1pthhqAr|0CCwwY64 z(nw=uN3U{>1I_3N^dQY-rY=#d1B51{dD=^H>nwic*5Ma%HhKqRK={VdNbab<{|puf z@#wktmJo#M|HS>^Zo{W>5boRLdQqaI=k?-3O%0e>I)2RFTaj2_AIJRcUU z!tbb#)VWYh+WYvHrjj4$W`%qagGbSAYhtMSH=#YTNUgPRhE=!VQv^qC5l=hL^W1y+U+L~ZBYh;uj#`c>w9 zRyP^t-{a@pi4-2=T~~eAVe$C`DV{Asd_iEJHCh$Ppb3qu+g{n5%hj^?#9+$TexqPIV zu)T1f|1@6rBrY-iZ5REjPczmAnF?p-riQGeJ~qU0yX6aSK2=zn!5v(QH=6P$X>1=Muc;jpwToO2jy7>cFg>_>=O!zi@&X`|yRfGt$`6#=; z=fSFQNwaYl!=v@E1W38CI30>vRGQ~?M1gH#$)(^TtSxR5?a$?HKAX?f_-1Cm8~tkj zfRafd4QCfx8;Ob`Br4+@6U@qBPd|B*F(BySK*#rA_fOrJh;<0!{SyAm9f4mt&)(R; z!hqq|-(Q`pK{3R#)maVE6 z3?jgxlODoxL1$Jm1IlslDajna264BwOjI_mJ7Gjg?XZZmiuuYd{3Q8AMyVA4`|Be# zVyaP9fk#_9eo}ZB;mSIU#iJYgVn{LSln)7KAyqz1k?qx2H!F?bX5URB9IplB zXe45dvTJ3nq@5M{uWjO-@}_~T>hAu}6Cp!IM~;p4s(reNI1wJ^L#s-rgQ1}xpsn~Z zS5n&9=W$agK|(Mah*Ho#y3JskBz5OK^!~PnA5Z-CDUC~1 z+XFcRViL~Xt{(1{M$@xEZ@l;Jj+du-*`ncL1W(2;oPO-qm`bo(p7(R2=w0VHemkf!=m!^;fedH}G$PNX{OYZIKS_C?p@=bFX}o zg|6%7G_zY{ZIf(P>h5@y!Mtb3%BHr2-tbsi#!ulaISRAQPm0d4H(&V}M6%7+nb^V9 zA0+q_e}$wgd&xr_ztM91vvq$U+)`eCXu%Ar2bVRtiMYM*_tp4vX(Q`uz2suI{NY6D z1Idb$x5UJZ7ceUnLrfbEJhB{D<%b(F3J9L?MO_q|$I5;R>Ppn7_MhH}E(MA0W!IDN zuY8-d(#==WthFk)*?;Sw<;I&<%ebSkNxpJG$}!x`JV{%i%NOB>ob|@SZp+uohF$2= z>J(yRyWyUIEUZzQ0n*9jFo*>K`O%P+r=>}%E;(F}t*c8On{rPoSIUH~)!d4Im|k~F zW84x7!!AX+aA)nwAIcRiyut#UlgW+0xoGrc-|jTMbo9dZZdhr{gg5}_{ZWitZ{X_H;z@HQ-7$-{*G&aenvPrNRUWW_|bI%!ne8+KM zD50cjRoM`0S~90TwJ*XVhmJob9@`#RS=g>?BDrgZY%ip}WsvZ*`jGKRTjZ^U6Zco? z#jILS33}?Pns1~WZ>K+u*mgTOY&3p^k<1$0Xt+?R#G%ZC*Xht@25Qdsp}mqLIjG`} zlcX)$#vB|Ipi#Dl=bdOvOpGL<2jM~Cx`dFZ?2dd zkzb?6m&lPB#WsI&4*5);OdaXA;`jo5WIAg;_|o=U>rV40EPbtKE4z_%MiT_RjQLC^ zRP->kSQx>YOxna61r()i#Y3<;3Sg<(YDGyqm^jf7Nmi->?{@rf zz8j}V0bk`C%I6E*G;^4~nSD3z~j-%+r?kwY1=1q|*v*60o! zH_fKFo$o3Z&yQ|e!~0$GSwG33Fza?G6XtugKvZM2_d?U~_~f_b6yXR*&_`UG06fAq zs@ii@=J3C5U(z`WY_&w>VwuQ~njBVqwXLbKD{@wlXKoI#5Fv%Bin8u!XQ}h!VuY7 zx%+B3M0_>*PkklN|Mi0K=G9OQ5jFrIn+^cH`H$k@pRyoDL&FZ2J%Q5fb>GswWeK|x_sj+uqW9jK^@_yXw@v8SEF1zT$;NH2`gM;^YIs9oh zRn)8DZpO>y=F0i!k$dZcMhVcr_u_v1aqIl@`sYt~w^nZRQL8ZNMo}SN0_nh`?x%~( zy`K;E^A;x!;(>t&hCFFngf;iQGNx-sdusC>o)2}cOBP|Z=}3*e5~iC~ZnK)SPZy6* zdk>(?m}|8pZgc5Y56_>+*Urur2I|S z^bQuw7d$x#yn?5mb@@2hzS}9$F9{Qyha){#wl*$`UXM>L9dBG;bmlUsx2Q-ZosqIsVx7is%oyyu=tnQb-oKR`EgnqpfRyxv~gyS*A{ zaBJ$2Uf^){t9675u&XBB81=N`Gv7-;^7N>O-MDK!U~s&$dN&_u>^s}gMIK}!ZXTk1 z2=EP=!F-ZHbX;qFA4vW|x?BwnYSbr7)c)fcg!tJz+%AjS`-P1(dz(oIi{h_uk1@It zj5Ev{!D7}8*@6~aXKQ=kAFY3PTUJH$SS0?u7$iG_%THx#=v!Cp<>j_mNqEtu3NIUp_*KCJ1R z>?VmZmw`&^nuh+x?VP3Y(9rdr6N_tzyi+rHc_-Iq4Z>VDQ#pS3tSMOHw@+b;D`TOf0I` zc2d=a$@Y8`Wo+y;Bb-soQ>W`xgBdsKviz`0QF8>@81bty351uVSuAM1{T5=VeG@Md zmts56G8X!qr6wiJg14a(4o7*7f$;|HHL|qtoi|0DJIs*$3T`OLdQMQ6$@_^UrdcG{P=8OOCRlT z4l7Q36n`4UH}>MJ<{sV&7{F;^Gr7kfOGO=5oYyL9VsoOTo_nHq;QjK{S}G}qzJWz? zQ`65Tx(|%u4={EQf?v<>Yl5TU&VS^l#&k5sAm~LHq+wWOFps*u4tcEB2Cc5a%!;X@ zN7-p(AvEja!V|WA!oyUpSN!W)qn}fhymd#%ihQekd-Zw6#aNTJWLcNiq(<(G170N7QVnPr*3-|T zg0&Ut{pxREe!22vlt7wuwc>JbYm5<=zDv&}K|bx0=o{jP0tsgj0?GsZ*=r`k-=CkN z4w+&4?VZzrq-2*(CIr%4sueZWJr(KsAs9H=VeD=@g4P;ioRah-b_|MRR*)&PZb0z2ADJ`+4yC^A zK2gH;C@=dGG?5++!^Ge{iWz@rN-UPnZw#D|L?NCJ0YM&^9z`SM&piQ9i;DUoXFI{3 z&=wH2?wA)hg?qL~dxB(?nqV%vu|!_4HI#D`-3jSL^YT3)7NOK`bC^taMNLYA`b( zF}L-FP$yEu@9rl&$T3k%Dp5<==~)%$yic(_O+<~fYM+%{TaiIwW-66zYRkmRY-?-E zXi3nyJvkNTV(NuaAYG(BJVzkp*8QBg6L7?8(heN3z4J%uNN?1i@#GqyA9Y=r9SA-w zaXtiNIZZ4_=&--~6*`W23W!F(s?nTu?m!LWeCAw)pwx0F>)O?*;Ff;dXSWP;wg>cB zTVazx!tZD{mr8*FdJGE+j@gY7c6l&^?UVC!)Gq+-;FhNDiv~#q-)>=0b-<@x6AjJ$ z3UsF3dM$Ti(ENqm zqk&&l9EzLXxG?&#dh4{?fnezRdqf(wud-x%Kw@g^REKQH?iFxw93B+8CKg>}+c+6O zGVazcBi^SzMky}@hCO3Y&obk~|KK!Ywlpj5m6pgZ4nVMFhW!f&yYx}J=Q%~y=j-Q# zCS>5B%<6kIgAftbHt0fqcGITs{DXi~u8vlo!aE`4~4 z3Uo>+Zb~>AtV2oyLKs;Ds=S1@L;9#!wvaF6)q3UWXK)FYz|i_x@IfN}a9-MJDIJsx zCcNf**GL9Z2|p9u#)9x(aLjB%ghUAPfgVNpd+&j-Jp5$844u{ZDXWn*YIg@kaJ zcu0;7DCr?e1Na;Y4C5U zzsi_!`^_rm0wS-Dc~uv}_k<3b5DiggS_|5Wf+5Rfje|K~B$l;n&UF{pbO^`VcCOf9 z>}F72_|`B;`b9!>4-`Qha&~-oOPqZ%sU=okhR8re4eq=7)#tqKoH9_x5tI2ZWcWSu zfK6BW0-fvJi&Y6Nnc}2vY7)OzTa%JU`ycAZEf(gvd@^LXLy#9S2s#Z9FOgc_DONb* z9z-pVhl)jTmJpX+frQRq+8T>U8)@P+Xcz(rmMOvHuY19^M3rDcfqJ(j3f%U@xVUMT z=NuNj zC5SPQ;RPCwN*^JWXd4wZFVmPtZ%{A~(Ekd#4UOW|ed*spnQFJhPb-Vrz_;Q25DtX( zs3iqdfooRK?0B~>Js%lQB1nFRoA~3BsN?ghauC^8cZ#X`At+jZ?F6Rv-PpOZp*}pk z=8@ykWeKCMJF-Uu+zrN^BfiZ5G21ey!zD1rPr}1jCCnk6DBQ$HZ<^4$Oni|7fXIB; zyZ=D+dR!KV9s_a1+?dG_KTK>q_0%-U7uNhENrQmA-DjzR_S|lr6c3gj-5edUOlTgW z?r$FgCr;*2;t_MJVge+hhQ;V_C^cb4bHs%3SOJJM%l!6V@N?-WbKh$1%7u z?)R3!#FIq(4IX{g9O%p)HzbRfnv@nZx=W`C{dh3zmK06dZJ?7A3QC4UnhxD?CGsKO< zrgjMiHy-iFw0&(*u@>$SCt|V1BOGkPVNrV68~yM#$k#14QMtkvc_cXKvr{Xmdf;(q zwzO2xAmB;1gt%2c3#JfuxxkXawt?oLf=&-;z1992kR2bJf&~{eC`GXL;D>Z7OTdfo zRg)(xM0?t#o2r@of#|jT0|B&6GrmeMysCI--o|K;vbnmkM&S>er=;NtN*zwS@G`?l zgC^}_)B-B7XyF1hd-~28IO*FM)K#vN-s61`BE!}dNP5BmBY2hz_xl^YtRfi}XL-(;1HEn%i9?9cCnemJ@ zX!>RKG>w`Ng>!!Fcx-xAld}Z#8we?z!N)#9%@@_gqmF7l)Gpqr7i#j2gc+MWC}li_ z;=FAVF1Il}$=dz-rlT%Y-R_Ws6Z~W|mI?~zU@wN>oO>-1B2SCp&M-U=X5D##-H59| zMwpSy;x3*hS>EL{IYF75q-8U=M7bui_#I^?kVnGhtR7^(=yUU41K_!Jn{6@$Z5*_5 zb&A$ob%OII_|K=h1iWe`5gHYnxr(@v^W0=z-7}$ub~wKrGFZtMUw(cBh%OE=(me1c zSGSFdweBZ50Y`6&36vhJl!-4FC&-v$7Jy_Z`BI=TRWvBIoJS=r0P=GVPoq))SI+~5 zvU2)H5&*j?issefw@{~dnKSB*g7!abo2&-NRi{f95tMDraEL(P8-ugGh{;l^;avnI zoH{8s=UN7p$wFm=?sDtiGZ5j9jwrYbQ881zp$14QXFRi@e&rkcE9j}-b`ym{Xj7YQ> zbJ?>Sa(Vg1Ki&%SsJTI%UBj)8l6elmu*)+b^ZjXNNF`Af;Qj zt{u=E5J{f>n8s6Rw2D$5tE|}JY3gD9Vvc(ij%2~IJr2@f&8?eqSD&w51sjWJ=jA4^ z$(p`XB{kr#HV6R`W?dFtoI|!M6Jh=YYMVm4uH7*XuCT+SJ{!ONsm@kW+ z)WzX#EsHp61Zqe@&~HuSZ{qEujd96K6f;d}FvmegH>N^J4;ekLP;Tn>A?_}q@>DW$%I?~_ z;U}W4jC`RH#m1b@kn)yg4D*N)93A{D3sx&~N3NU!{x8umqiiO1Zk{LNtufho{GgZkO%+iiP_O@-KmK}7N6;r3N&jC$ox zL(afFZuKd=Q0cMnmnSFB4nMwlG8fS` z&%JXk0GqoMSe@~+U*TnA4stkNm5 znkE&_PaQY`!4gHN9Mq#5#v4049HLOs=4*NW1(_oOp((7GE5#oSlxXud=In>&V|n=VK}1hXrAaZ! z_=(%00)T#jR)o0{aVeHW2xvf!p-%~_xfCNK7AkJ0+HsR{@K*BoYk zxM!LC>V#py4gWP9P_j8Q_cjDh6s#WAy34~67=;qp7P)5l|79cS;rUn=@l->V@#WSKZ(2nX3a5K9FSH=<8>0#)WR4WNBjzw)1&$(t|G3i?)g+p^aT`9hU=~r1sTvO zTV49#D}0$`Gy*qne&OIR>r<@H;SVd5At*}?#^KK9A>c)bxUufZt8~Mwe(Kzb#H=t% zoTZ&VS?odTG$~U-D}kOaQZx~IiNn2>8cEfIwGf|1XJh&&yf5}2z-@f%Lh62n@Se)A ziRB;+eX7LI&;3HpFnCtzpbA`izX1gbKyqj1>Aw*6?ZQn;0zz1`NgZY<@K>w7uijk~8Jr2c40%7x2H`p-dN4smgE0 zni!VhlEi_h;~+_L;QW#g?0Im3*hPzBviVniP)YfFfw>CN)r|bVM~XoN5TWY&IftE4 zzH8N6+%Jg~#H=!GniZrnnP{y6Y{2%0JFedrHsMQ=!j4A|l{&U*)C2fg6x#=z{CEpnyd&w;j>jc)psiU=wXY&SX)9oY*@d!b)=7iOf# z28s}?j9i3~Jh(3J|4QtL7P_cLqqls{d|_0SNi{b^7v+;eD>Y#1aj)6^ z{y5vP9bZMxH84yIHgoBRniwZk?}q2Ylg>?A8_Ui`17u4 z@(P31%()HD25K5qFkC8v_|oP#EUrp2pOo~@?ICsUX3c~+5EO)}PR$eKA{j#ME!9Jk z=1Vep5yox0wo+raWbVCrvd+$9o2bGME*>qN=FAxulAWX%q#K_X(|C^vN0pEo!!d_% zhjr@{O1^mkchW&Oj?WT)Q7Mn0!6%MT0JrNe;EnZGbN?}KvNuXAv9Xk9;#;+}+#8Yy+?Se9A0M}2EnBJLo_706^Gsh^gnXy} zG>U6bOdp$`hE+8|geOZxV~mI>6^h6b3yhT^QjWgqi0{KBK)2^I!WWUmz!j%l%ty9I z!A(gt7 zAIn4L@R`(`GJ(K!!C&*djM1|wK+eOlJIC*tE2?(*?j4~B#KiL`eN9^|ZY0=V-2j2> z?&F9v2tASRh9ka146LpYq_Anv`X*?Q*wgWe^9J8FjutXR$UD%18d6E9w2nU0;N6zj z#0|2QI8+|^H{Vk+6{u~cc|jfJZ~BaYluiuK{LiygsfI)yV)Ye=pX(ewii$sr6xzP= z4db`kFwOTt(GU}`dI(HG2Fl<_ke6Z)XC0UgNe_N5(tWS+qcB!Pjj~4-L2gc5(2*I1 zZ<^yY@wF1nk5r0DUrBjnd$~OfmOg$nNCM`BG9vv4I(f!$4kpZYnOHJ0-A++xMfM;< z8Gbn8l2|2-Pu=Km$&2{CXpWQjwqDC*lObJQYYN~vTyqItLzWL2OfuZ$PUJFfVrOPX z`qBf(%JGQGs-{{}Jx%SrxSHJO-h0F1A#b>LMdi>T?HANdapn``6VSYg@-gU7#YPTJ zWum`mn1JF)EqJpAzVMlwWYua%P&h?fj2M@+j-b!y9fko1)^UszhZs#6t;1@&cqz;D z%o}>0#oX7&-0!Hp6Qe4NFvFFklvvUA>U%7%R|MD_a(^Jg^tZ-}{Ji z2K-iE)G~FG-i7LRPkuht&QCqC$$%9qT$9u<{80*D_2U4805K_^lF*fc=Pl(jtv`O< z_FEy7Z6q--Zf1Y}ddjk1V|eH{--fQzGGz9DTvcmhk8GXInYqIP;G&&RVIL9`+vR^G zI?Qam8^7iHd@hI5LF`2Qj$H@H{1#4RC!ier5G`pE{*}=@yx#lbKA|K1j*BJ!0Hx!- zUcIBI@Eto(*p^4E{}#h-g2mUrgO1E)fezavgAUv4Tj_nawX(Kj(6P3DiQk~Ml>a?q z56W#oIZ3M@jL6L-XM`aq8b%TdnXwyi)fohozy=deq!E3+sd5rGoTKxDax@2a2^8oq z3*NNlCKtWIA`PQCZ(^Ji@fLOTl+4K%HcPeh!2OFk=#jTyirQTR$yW3>mG+Nt>k(Ue z`CpMppwq`Sk?ku93NPtG+eFUOJKzvh8qHz0-NYdpWawtE5f9hsR(c3An|2&8i;nNCEQBpdlGkn}` zamE+Yhm)rpeOEQcU|7? zn9|cL7_AZ~)tF>V@aetPzykufb2fbX*-y4hhAaD6LkA+B4&jsWa)W~(&RWDqjTNKU zZ|i?BfIRe{$Dv6i^!D9A*wzCbiAMchpXgaxS{mrtn^;-?s!l?p+ASAenr)!25d)6H z4ok!)h-}J%2xDqUN5|hjQ3#TblO)m#lsgfo(#{5GzS?a|2j_drVe#Vlu8il+p{<$V zDxU~m(aW|u+|krrGEhLJtZu2%^%~v|oDFF>FS))qr9C*#CNpFRz7k{4Xu4uW(3Yoz zpkmoxtpBU{yz()u`k)%% z<|sAME7u%X_YX*C(9Upa5#`a59kh-br>E_D@5>Qog6_oU_Y}u z>t)cE1i6X~G)nN>RW=R=wl4BIx}f9h4FCN9>mX}LB$JgdBeMUE_#;vuH&a9q5jrXv zUA{^oVlfx@HlXPClLZ=-!s7|6unA4B5c^8w#f{X7Vsf;3Gl?+?x*=%_R2Rf%A-b~B z=2rbF+B)`NnSQZgUmqaAe7css=!W3(bO@2o8Sx7_Noci(c)=Qin4w`Igng>GNk5Rr zK~ggBOYC>BvWi?8$!_h;{4CD`t;DrD;n1JR#I6-b3#SoYovyylg&$4iOs>Kz zxmAAk0p-9-+*YK2Qsyp=M-Ht3NyDb?Dl^s74rUe;v)!C}}`HV6guSo7bOUSyT>(EcK_oJ34(=_T>Dohd;MWo547f`8KRef3# z=eVE4%{-r!9%26m85lSn=pymI_pkqL)4xCeOE>#xiGNq{@13-N3%-2Tf-0GR>8t%y z@ZY-&|5b1ibb9{(-D~)#o