Skip to content

Commit 9b4724a

Browse files
spiegel-im-spiegelspiegel-im-spiegel
committedJul 18, 2023
Fixed a bug that caused an error when registering duplicate URLs
·
v0.12.5v0.10.4
1 parent 53660df commit 9b4724a

File tree

5 files changed

+33
-25
lines changed

5 files changed

+33
-25
lines changed
 

‎db/webpage-repository.go

+10-7
Original file line numberDiff line numberDiff line change
@@ -31,17 +31,20 @@ func (repos *Repository) FindWebpageByURL(ctx context.Context, url string) (*mod
3131
}
3232

3333
// InsertAPODData method inserts APOD data to database.
34-
func (repos *Repository) InsertWebpage(ctx context.Context, data []model.Webpage) error {
34+
func (repos *Repository) InsertWebpage(ctx context.Context, datalist []model.Webpage) error {
3535
if repos == nil {
3636
return errs.Wrap(ecode.ErrNullPointer)
3737
}
38-
if err := repos.Db().WithContext(ctx).Transaction(func(tx *gorm.DB) error {
39-
if t := tx.Create(data); t.Error != nil {
40-
return errs.Wrap(t.Error)
38+
for _, data := range datalist {
39+
data := data
40+
if err := repos.Db().WithContext(ctx).Transaction(func(tx *gorm.DB) error {
41+
if t := tx.Create(&data); t.Error != nil {
42+
return errs.Wrap(t.Error)
43+
}
44+
return nil
45+
}); err != nil {
46+
return err
4147
}
42-
return nil
43-
}); err != nil {
44-
return err
4548
}
4649
return nil
4750
}

‎dependency.png

17.1 KB
Loading

‎go.mod

+6-6
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ go 1.20
44

55
require (
66
github.com/PuerkitoBio/goquery v1.8.1
7-
github.com/bluesky-social/indigo v0.0.0-20230706164817-fdb5f9774705
8-
github.com/glebarez/sqlite v1.8.0
7+
github.com/bluesky-social/indigo v0.0.0-20230714174244-57d75d8cfc65
8+
github.com/glebarez/sqlite v1.9.0
99
github.com/goark/errs v1.3.2
1010
github.com/goark/errs/zapobject v0.1.3
1111
github.com/goark/fetch v0.4.1
@@ -22,7 +22,7 @@ require (
2222
go.uber.org/zap v1.24.0
2323
golang.org/x/image v0.9.0
2424
golang.org/x/net v0.12.0
25-
gorm.io/gorm v1.25.1
25+
gorm.io/gorm v1.25.2
2626
moul.io/zapgorm2 v1.3.0
2727
)
2828

@@ -31,7 +31,7 @@ require (
3131
github.com/atotto/clipboard v0.1.4 // indirect
3232
github.com/dustin/go-humanize v1.0.1 // indirect
3333
github.com/fsnotify/fsnotify v1.6.0 // indirect
34-
github.com/glebarez/go-sqlite v1.21.1 // indirect
34+
github.com/glebarez/go-sqlite v1.21.2 // indirect
3535
github.com/gogo/protobuf v1.3.2 // indirect
3636
github.com/google/uuid v1.3.0 // indirect
3737
github.com/gorilla/websocket v1.5.0 // indirect
@@ -94,8 +94,8 @@ require (
9494
gopkg.in/ini.v1 v1.67.0 // indirect
9595
gopkg.in/yaml.v3 v3.0.1 // indirect
9696
lukechampine.com/blake3 v1.1.7 // indirect
97-
modernc.org/libc v1.22.3 // indirect
97+
modernc.org/libc v1.22.5 // indirect
9898
modernc.org/mathutil v1.5.0 // indirect
9999
modernc.org/memory v1.5.0 // indirect
100-
modernc.org/sqlite v1.21.1 // indirect
100+
modernc.org/sqlite v1.23.1 // indirect
101101
)

‎go.sum

+12-12
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
4646
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
4747
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
4848
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
49-
github.com/bluesky-social/indigo v0.0.0-20230706164817-fdb5f9774705 h1:MK11a0stoFLXKVbAlvsZUZPgf12skLJKNUEBnTTVyaE=
50-
github.com/bluesky-social/indigo v0.0.0-20230706164817-fdb5f9774705/go.mod h1:oDI5NiD0XzShv5VITWyUJNP3pSh4prTDEzhKbkdKORA=
49+
github.com/bluesky-social/indigo v0.0.0-20230714174244-57d75d8cfc65 h1:0z5rF7oA9eqqoAFoiHTfioBK8U4hzMmyGMoajEqJVFE=
50+
github.com/bluesky-social/indigo v0.0.0-20230714174244-57d75d8cfc65/go.mod h1:oDI5NiD0XzShv5VITWyUJNP3pSh4prTDEzhKbkdKORA=
5151
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
5252
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
5353
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -73,10 +73,10 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
7373
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
7474
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
7575
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
76-
github.com/glebarez/go-sqlite v1.21.1 h1:7MZyUPh2XTrHS7xNEHQbrhfMZuPSzhkm2A1qgg0y5NY=
77-
github.com/glebarez/go-sqlite v1.21.1/go.mod h1:ISs8MF6yk5cL4n/43rSOmVMGJJjHYr7L2MbZZ5Q4E2E=
78-
github.com/glebarez/sqlite v1.8.0 h1:02X12E2I/4C1n+v90yTqrjRa8yuo7c3KeHI3FRznCvc=
79-
github.com/glebarez/sqlite v1.8.0/go.mod h1:bpET16h1za2KOOMb8+jCp6UBP/iahDpfPQqSaYLTLx8=
76+
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
77+
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
78+
github.com/glebarez/sqlite v1.9.0 h1:Aj6bPA12ZEx5GbSF6XADmCkYXlljPNUY+Zf1EQxynXs=
79+
github.com/glebarez/sqlite v1.9.0/go.mod h1:YBYCoyupOao60lzp1MVBLEjZfgkq0tdB1voAQ09K9zw=
8080
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
8181
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
8282
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -732,8 +732,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
732732
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
733733
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
734734
gorm.io/gorm v1.23.6/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk=
735-
gorm.io/gorm v1.25.1 h1:nsSALe5Pr+cM3V1qwwQ7rOkw+6UeLrX5O4v3llhHa64=
736-
gorm.io/gorm v1.25.1/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
735+
gorm.io/gorm v1.25.2 h1:gs1o6Vsa+oVKG/a9ElL3XgyGfghFfkKA2SInQaCyMho=
736+
gorm.io/gorm v1.25.2/go.mod h1:L4uxeKpfBml98NYqVqwAdmV1a2nBtAec/cf3fpucW/k=
737737
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
738738
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
739739
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -743,14 +743,14 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
743743
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
744744
lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
745745
lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
746-
modernc.org/libc v1.22.3 h1:D/g6O5ftAfavceqlLOFwaZuA5KYafKwmr30A6iSqoyY=
747-
modernc.org/libc v1.22.3/go.mod h1:MQrloYP209xa2zHome2a8HLiLm6k0UT8CoHpV74tOFw=
746+
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
747+
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
748748
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
749749
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
750750
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
751751
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
752-
modernc.org/sqlite v1.21.1 h1:GyDFqNnESLOhwwDRaHGdp2jKLDzpyT/rNLglX3ZkMSU=
753-
modernc.org/sqlite v1.21.1/go.mod h1:XwQ0wZPIh1iKb5mkvCJ3szzbhk+tykC8ZWqTRTgYRwI=
752+
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
753+
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
754754
moul.io/zapgorm2 v1.3.0 h1:+CzUTMIcnafd0d/BvBce8T4uPn6DQnpIrz64cyixlkk=
755755
moul.io/zapgorm2 v1.3.0/go.mod h1:nPVy6U9goFKHR4s+zfSo1xVFaoU7Qgd5DoCdOfzoCqs=
756756
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

‎webpage/feed.go

+5
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,12 @@ func (cfg *Config) getNewDataList(ctx context.Context, items []*feed.Item) {
4040
if cfg.itemPool == nil {
4141
cfg.CreatePool()
4242
}
43+
urls := map[string]bool{}
4344
for _, item := range items {
45+
if urls[item.Link] {
46+
continue
47+
}
48+
urls[item.Link] = true
4449
if page, err := cfg.find(ctx, item.Link); err != nil || page == nil {
4550
cfg.itemPool.putFeedItem(ctx, item)
4651
cfg.Logger().Debug("new item", zap.Any("item", item))

0 commit comments

Comments
 (0)