From 11a0184727378d4b26fc4b07dbe293662546d6fc Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 1 Feb 2025 11:29:51 +0100 Subject: [PATCH 1/4] Save unknown federations to database If there are unknown federations, we still save it to database, so user could find them using api. This also silent warning for NON federation. --- .../crawler/src/main/scala/Downloader.scala | 32 +++++++++++-------- .../src/test/scala/DownloaderTest.scala | 2 +- modules/domain/src/main/scala/Domain.scala | 6 ++-- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/modules/crawler/src/main/scala/Downloader.scala b/modules/crawler/src/main/scala/Downloader.scala index 002be55..6471b44 100644 --- a/modules/crawler/src/main/scala/Downloader.scala +++ b/modules/crawler/src/main/scala/Downloader.scala @@ -40,21 +40,31 @@ object Downloader: .ifM(parse(line), none.pure[IO]) .handleErrorWith(e => error"Error while parsing line: $line, error: $e".as(none)) - // shamelessly copied (with some minor modificaton) from: https://github.com/lichess-org/lila/blob/8033c4c5a15cf9bb2b36377c3480f3b64074a30f/modules/fide/src/main/FidePlayerSync.scala#L131 def parse(line: String)(using Logger[IO]): IO[Option[(NewPlayer, Option[NewFederation])]] = + parsePlayer(line).traverse(x => findFederation(x._1, x._2)) + + def findFederation(player: NewPlayer, federationId: Option[FederationId])(using + Logger[IO] + ): IO[(NewPlayer, Option[NewFederation])] = + def f(id: FederationId, playerId: PlayerId): IO[Option[NewFederation]] = + if id.value.toLowerCase == "non" then None.pure + else + Federation.all + .get(id) + .fold( + warn"cannot find federation: $id for player: $playerId" *> NewFederation(id, id.value).some.pure + )(name => NewFederation(id, name).some.pure) + + federationId.traverse(f(_, player.id)).map(fed => (player, fed.flatten)) + + // shamelessly copied (with some minor modificaton) from: https://github.com/lichess-org/lila/blob/8033c4c5a15cf9bb2b36377c3480f3b64074a30f/modules/fide/src/main/FidePlayerSync.scala#L131 + def parsePlayer(line: String)(using Logger[IO]): Option[(NewPlayer, Option[FederationId])] = def string(start: Int, end: Int): Option[String] = line.substring(start, end).trim.some.filter(_.nonEmpty) def number(start: Int, end: Int): Option[Int] = string(start, end).flatMap(_.toIntOption) def rating(start: Int, end: Int): Option[Rating] = string(start, end) >>= Rating.fromString - def findFed(id: FederationId, playerId: PlayerId): IO[Option[NewFederation]] = - Federation - .nameById(id) - .fold(warn"cannot find federation: $id for player: $playerId" *> none[NewFederation].pure[IO])(name => - NewFederation(id, name).some.pure[IO] - ) - - val x = for + for id <- number(0, 15) >>= PlayerId.option name <- string(15, 76).map(_.filterNot(_.isDigit).trim) if name.sizeIs > 2 @@ -79,10 +89,6 @@ object Downloader: active = inactiveFlag.isEmpty ) -> federationId - x.traverse: - case (player, Some(fedId)) => findFed(fedId, player.id).map(fed => (player, fed)) - case (player, None) => (player, none).pure[IO] - object Decompressor: import de.lhns.fs2.compress.* diff --git a/modules/crawler/src/test/scala/DownloaderTest.scala b/modules/crawler/src/test/scala/DownloaderTest.scala index 87526fe..99a1785 100644 --- a/modules/crawler/src/test/scala/DownloaderTest.scala +++ b/modules/crawler/src/test/scala/DownloaderTest.scala @@ -23,5 +23,5 @@ object DownloaderTest extends SimpleIOSuite: .fold[Set[FederationId]](Set.empty)((acc, x) => acc ++ x.map(_.id)) .compile .last - .map(x => Federation.names.keySet.diff(x.get)) + .map(x => Federation.all.keySet.diff(x.get)) .map(x => expect(x == Set.empty)) diff --git a/modules/domain/src/main/scala/Domain.scala b/modules/domain/src/main/scala/Domain.scala index 82fa96e..3ac8d29 100644 --- a/modules/domain/src/main/scala/Domain.scala +++ b/modules/domain/src/main/scala/Domain.scala @@ -131,10 +131,12 @@ case class Federation( object Federation: - def nameById(id: FederationId): Option[String] = names.get(id) + def nameById(id: FederationId): Option[String] = + if id.value.toLowerCase == "non" then None + else all.get(id).orElse(id.value.some) import io.github.iltotore.iron.* - val names: Map[FederationId, String] = Map( + val all: Map[FederationId, String] = Map( FederationId("AFG") -> "Afghanistan", FederationId("AHO") -> "Netherlands Antilles", FederationId("ALB") -> "Albania", From 2a3783f49236f1fe6ba4cda3f7ad1b186515fab7 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 1 Feb 2025 11:58:30 +0100 Subject: [PATCH 2/4] Clean up Downloader --- .../crawler/src/main/scala/Downloader.scala | 32 ++++++++----------- .../crawler/src/test/scala/ParserTest.scala | 2 +- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/modules/crawler/src/main/scala/Downloader.scala b/modules/crawler/src/main/scala/Downloader.scala index 6471b44..3e25760 100644 --- a/modules/crawler/src/main/scala/Downloader.scala +++ b/modules/crawler/src/main/scala/Downloader.scala @@ -35,30 +35,24 @@ object Downloader: .evalMap(parseLine) .collect { case Some(x) => x } - def parseLine(line: String)(using Logger[IO]): IO[Option[(NewPlayer, Option[NewFederation])]] = - IO(line.trim.nonEmpty) - .ifM(parse(line), none.pure[IO]) - .handleErrorWith(e => error"Error while parsing line: $line, error: $e".as(none)) + def parseLine(line: String): Logger[IO] ?=> IO[Option[(NewPlayer, Option[NewFederation])]] = - def parse(line: String)(using Logger[IO]): IO[Option[(NewPlayer, Option[NewFederation])]] = - parsePlayer(line).traverse(x => findFederation(x._1, x._2)) + inline def parse(line: String): IO[Option[(NewPlayer, Option[NewFederation])]] = + parsePlayer(line).traverse: (player, federationId) => + federationId.flatTraverse(findFederation(_, player.id)).map(fed => (player, fed)) - def findFederation(player: NewPlayer, federationId: Option[FederationId])(using - Logger[IO] - ): IO[(NewPlayer, Option[NewFederation])] = - def f(id: FederationId, playerId: PlayerId): IO[Option[NewFederation]] = - if id.value.toLowerCase == "non" then None.pure - else - Federation.all - .get(id) - .fold( - warn"cannot find federation: $id for player: $playerId" *> NewFederation(id, id.value).some.pure - )(name => NewFederation(id, name).some.pure) + IO(line.trim.nonEmpty) + .ifM(parse(line), none.pure[IO]) + .handleErrorWith(e => Logger[IO].error(e)(s"Error while parsing line: $line").as(none)) - federationId.traverse(f(_, player.id)).map(fed => (player, fed.flatten)) + def findFederation(id: FederationId, playerId: PlayerId): Logger[IO] ?=> IO[Option[NewFederation]] = + Federation.nameById(id) match + case None => + warn"cannot find federation: $id for player: $playerId" *> NewFederation(id, id.value).some.pure + case Some(name) => NewFederation(id, name).some.pure // shamelessly copied (with some minor modificaton) from: https://github.com/lichess-org/lila/blob/8033c4c5a15cf9bb2b36377c3480f3b64074a30f/modules/fide/src/main/FidePlayerSync.scala#L131 - def parsePlayer(line: String)(using Logger[IO]): Option[(NewPlayer, Option[FederationId])] = + def parsePlayer(line: String): Option[(NewPlayer, Option[FederationId])] = def string(start: Int, end: Int): Option[String] = line.substring(start, end).trim.some.filter(_.nonEmpty) def number(start: Int, end: Int): Option[Int] = string(start, end).flatMap(_.toIntOption) diff --git a/modules/crawler/src/test/scala/ParserTest.scala b/modules/crawler/src/test/scala/ParserTest.scala index 5783a36..f9c1964 100644 --- a/modules/crawler/src/test/scala/ParserTest.scala +++ b/modules/crawler/src/test/scala/ParserTest.scala @@ -31,4 +31,4 @@ object ParserTest extends SimpleIOSuite: .map(_.flatten.map(_.active)) .map(x => expect(x == List(false, false, true, true))) - private def parse(s: String) = Downloader.parse(s).map(_.map(_._1)) + private def parse(s: String) = Downloader.parseLine(s).map(_.map(_._1)) From 3010f83e6886856c271dfc04f81ac90c05dadb3d Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 1 Feb 2025 12:04:18 +0100 Subject: [PATCH 3/4] Code golf Thanks Max (a.k.a Masynchin) --- modules/crawler/src/main/scala/Downloader.scala | 2 +- modules/crawler/src/test/scala/DownloaderTest.scala | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/crawler/src/main/scala/Downloader.scala b/modules/crawler/src/main/scala/Downloader.scala index 3e25760..2c51c48 100644 --- a/modules/crawler/src/main/scala/Downloader.scala +++ b/modules/crawler/src/main/scala/Downloader.scala @@ -33,7 +33,7 @@ object Downloader: .through(fs2.text.lines) .drop(1) // first line is header .evalMap(parseLine) - .collect { case Some(x) => x } + .unNone def parseLine(line: String): Logger[IO] ?=> IO[Option[(NewPlayer, Option[NewFederation])]] = diff --git a/modules/crawler/src/test/scala/DownloaderTest.scala b/modules/crawler/src/test/scala/DownloaderTest.scala index 99a1785..dff3f91 100644 --- a/modules/crawler/src/test/scala/DownloaderTest.scala +++ b/modules/crawler/src/test/scala/DownloaderTest.scala @@ -19,9 +19,9 @@ object DownloaderTest extends SimpleIOSuite: .build .use: client => Downloader(client).fetch - .map(x => x._2) - .fold[Set[FederationId]](Set.empty)((acc, x) => acc ++ x.map(_.id)) + .map(x => x._2.map(_.id)) + .unNone .compile - .last - .map(x => Federation.all.keySet.diff(x.get)) + .to(Set) + .map(Federation.all.keySet.diff) .map(x => expect(x == Set.empty)) From 2fd0043780da88d2e945bce5a5d00775be07a33d Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Sat, 1 Feb 2025 12:20:18 +0100 Subject: [PATCH 4/4] Better filter bird year --- modules/crawler/src/main/scala/Downloader.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/crawler/src/main/scala/Downloader.scala b/modules/crawler/src/main/scala/Downloader.scala index 2c51c48..7e2296f 100644 --- a/modules/crawler/src/main/scala/Downloader.scala +++ b/modules/crawler/src/main/scala/Downloader.scala @@ -16,6 +16,7 @@ trait Downloader: object Downloader: val downloadUrl = uri"http://ratings.fide.com/download/players_list.zip" + val currentYear = java.time.Year.now.getValue lazy val request = Request[IO]( method = Method.GET, @@ -66,7 +67,7 @@ object Downloader: wTitle = string(89, 94) >>= Title.apply otherTitles = string(94, 109).fold(Nil)(OtherTitle.applyToList) sex = string(79, 82) >>= Sex.apply - year = number(152, 156).filter(_ > 1000) + year = number(152, 156).filter(y => y > 1000 && y < currentYear) inactiveFlag = string(158, 160).filter(_.contains("i")) federationId = string(76, 79) >>= FederationId.option yield NewPlayer(