From 551aa7e788e7c485bb66f83783eed6d4f3e9549c Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Fri, 31 Jan 2025 16:01:08 +0100 Subject: [PATCH 1/2] Add warning log when crawler finds new federations --- modules/crawler/src/main/scala/Crawler.scala | 20 +++++++++++---- .../crawler/src/test/scala/CrawlerTest.scala | 25 +++++++++++-------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/modules/crawler/src/main/scala/Crawler.scala b/modules/crawler/src/main/scala/Crawler.scala index 1014e56..d8c7949 100644 --- a/modules/crawler/src/main/scala/Crawler.scala +++ b/modules/crawler/src/main/scala/Crawler.scala @@ -67,16 +67,22 @@ object Downloader: // shamelessly copied (with some minor modificaton) from: https://github.com/lichess-org/lila/blob/8033c4c5a15cf9bb2b36377c3480f3b64074a30f/modules/fide/src/main/FidePlayerSync.scala#L131 def parseLine(line: String)(using Logger[IO]): IO[Option[(NewPlayer, Option[NewFederation])]] = - IO(parse(line)) - .handleErrorWith(e => error"Error while parsing line: $line, error: $e".as(none)) + parse(line).handleErrorWith(e => error"Error while parsing line: $line, error: $e".as(none)) - def parse(line: String): Option[(NewPlayer, Option[NewFederation])] = + def parse(line: String)(using Logger[IO]): IO[Option[(NewPlayer, Option[NewFederation])]] = def string(start: Int, end: Int): Option[String] = line.substring(start, end).trim.some.filter(_.nonEmpty) def number(start: Int, end: Int): Option[Int] = string(start, end).flatMap(_.toIntOption) def rating(start: Int, end: Int): Option[Rating] = string(start, end) >>= Rating.fromString - for + def findFed(id: FederationId, playerId: PlayerId): IO[Option[NewFederation]] = + Federation + .nameById(id) + .fold(warn"cannot find federation: $id for player: $playerId" *> none[NewFederation].pure[IO])(name => + NewFederation(id, name).some.pure[IO] + ) + + val x = for id <- number(0, 15) >>= PlayerId.option name <- string(15, 76).map(_.filterNot(_.isDigit).trim) if name.sizeIs > 2 @@ -99,7 +105,11 @@ object Downloader: sex = sex, birthYear = year, active = inactiveFlag.isEmpty - ) -> federationId.flatMap(id => Federation.nameById(id).map(NewFederation(id, _))) + ) -> federationId + + x.traverse: + case (player, Some(fedId)) => findFed(fedId, player.id).map(fed => (player, fed)) + case (player, None) => (player, none).pure[IO] object Decompressor: diff --git a/modules/crawler/src/test/scala/CrawlerTest.scala b/modules/crawler/src/test/scala/CrawlerTest.scala index f8aca13..e5f5a6b 100644 --- a/modules/crawler/src/test/scala/CrawlerTest.scala +++ b/modules/crawler/src/test/scala/CrawlerTest.scala @@ -2,18 +2,21 @@ package fide package crawler package test +import cats.effect.IO import cats.syntax.all.* -import fide.domain.{ NewPlayer, OtherTitle } +import fide.domain.OtherTitle +import org.typelevel.log4cats.Logger import weaver.* -object CrawlerTest extends FunSuite: +object CrawlerTest extends SimpleIOSuite: + + given Logger[IO] = org.typelevel.log4cats.noop.NoOpLogger[IO] test("player with other titles"): - val x = - parse: - "10001492 Ojok, Patrick UGA M FI,LSI 1638 0 20 1932 0 20 1926 0 20 1974 " - .get - expect(x.otherTitles == List(OtherTitle.FI, OtherTitle.LSI)) + parse( + "10001492 Ojok, Patrick UGA M FI,LSI 1638 0 20 1932 0 20 1926 0 20 1974 " + ).map(_.get.otherTitles) + .map(x => expect(x == List(OtherTitle.FI, OtherTitle.LSI))) test("active flag"): val lines = List( @@ -22,7 +25,9 @@ object CrawlerTest extends FunSuite: "29976634 Aafrin, S F Aja SRI F 2012 w ", "1478800 Aagaard, Christian DEN M 1999 " ) - val actives = lines.traverse(parse).get.map(_.active) - expect(actives == List(false, false, false, true)) + lines + .traverse(parse) + .map(_.flatten.map(_.active)) + .map(x => expect(x == List(false, false, false, true))) - private def parse(s: String): Option[NewPlayer] = Downloader.parse(s).map(_._1) + private def parse(s: String) = Downloader.parse(s).map(_.map(_._1)) From b400e5599f4c0822186d958b636156ab98fb03f3 Mon Sep 17 00:00:00 2001 From: Thanh Le Date: Fri, 31 Jan 2025 17:28:46 +0100 Subject: [PATCH 2/2] Add missing federations --- modules/domain/src/main/scala/Domain.scala | 361 +++++++++++---------- 1 file changed, 182 insertions(+), 179 deletions(-) diff --git a/modules/domain/src/main/scala/Domain.scala b/modules/domain/src/main/scala/Domain.scala index bcd5ec2..82fa96e 100644 --- a/modules/domain/src/main/scala/Domain.scala +++ b/modules/domain/src/main/scala/Domain.scala @@ -135,207 +135,210 @@ object Federation: import io.github.iltotore.iron.* val names: Map[FederationId, String] = Map( - FederationId("FID") -> "FIDE", - FederationId("USA") -> "United States of America", - FederationId("IND") -> "India", - FederationId("CHN") -> "China", - FederationId("RUS") -> "Russia", - FederationId("AZE") -> "Azerbaijan", - FederationId("FRA") -> "France", - FederationId("UKR") -> "Ukraine", - FederationId("ARM") -> "Armenia", - FederationId("GER") -> "Germany", - FederationId("ESP") -> "Spain", - FederationId("NED") -> "Netherlands", - FederationId("HUN") -> "Hungary", - FederationId("POL") -> "Poland", - FederationId("ENG") -> "England", - FederationId("ROU") -> "Romania", - FederationId("NOR") -> "Norway", - FederationId("UZB") -> "Uzbekistan", - FederationId("ISR") -> "Israel", - FederationId("CZE") -> "Czech Republic", - FederationId("SRB") -> "Serbia", - FederationId("CRO") -> "Croatia", - FederationId("GRE") -> "Greece", - FederationId("IRI") -> "Iran", - FederationId("TUR") -> "Turkiye", - FederationId("SLO") -> "Slovenia", + FederationId("AFG") -> "Afghanistan", + FederationId("AHO") -> "Netherlands Antilles", + FederationId("ALB") -> "Albania", + FederationId("ALG") -> "Algeria", + FederationId("AND") -> "Andorra", + FederationId("ANG") -> "Angola", + FederationId("ANT") -> "Antigua and Barbuda", FederationId("ARG") -> "Argentina", - FederationId("SWE") -> "Sweden", - FederationId("GEO") -> "Georgia", - FederationId("ITA") -> "Italy", - FederationId("CUB") -> "Cuba", - FederationId("AUT") -> "Austria", - FederationId("PER") -> "Peru", - FederationId("BUL") -> "Bulgaria", - FederationId("BRA") -> "Brazil", - FederationId("DEN") -> "Denmark", - FederationId("SUI") -> "Switzerland", - FederationId("CAN") -> "Canada", - FederationId("SVK") -> "Slovakia", - FederationId("LTU") -> "Lithuania", - FederationId("VIE") -> "Vietnam", + FederationId("ARM") -> "Armenia", + FederationId("ARU") -> "Aruba", FederationId("AUS") -> "Australia", + FederationId("AUT") -> "Austria", + FederationId("AZE") -> "Azerbaijan", + FederationId("BAH") -> "Bahamas", + FederationId("BAN") -> "Bangladesh", + FederationId("BAR") -> "Barbados", + FederationId("BDI") -> "Burundi", FederationId("BEL") -> "Belgium", - FederationId("MNE") -> "Montenegro", - FederationId("MDA") -> "Moldova", - FederationId("KAZ") -> "Kazakhstan", - FederationId("ISL") -> "Iceland", - FederationId("COL") -> "Colombia", + FederationId("BER") -> "Bermuda", + FederationId("BHU") -> "Bhutan", FederationId("BIH") -> "Bosnia & Herzegovina", - FederationId("EGY") -> "Egypt", - FederationId("FIN") -> "Finland", - FederationId("MGL") -> "Mongolia", - FederationId("PHI") -> "Philippines", + FederationId("BIZ") -> "Belize", FederationId("BLR") -> "Belarus", - FederationId("LAT") -> "Latvia", - FederationId("POR") -> "Portugal", + FederationId("BOL") -> "Bolivia", + FederationId("BOT") -> "Botswana", + FederationId("BRA") -> "Brazil", + FederationId("BRN") -> "Bahrain", + FederationId("BRU") -> "Brunei Darussalam", + FederationId("BUL") -> "Bulgaria", + FederationId("BUR") -> "Burkina Faso", + FederationId("CAF") -> "Central African Republic", + FederationId("CAM") -> "Cambodia", + FederationId("CAN") -> "Canada", + FederationId("CAY") -> "Cayman Islands", + FederationId("CGO") -> "Congo", + FederationId("CHA") -> "Chad", FederationId("CHI") -> "Chile", - FederationId("MEX") -> "Mexico", - FederationId("MKD") -> "North Macedonia", - FederationId("INA") -> "Indonesia", - FederationId("PAR") -> "Paraguay", - FederationId("EST") -> "Estonia", - FederationId("SGP") -> "Singapore", - FederationId("SCO") -> "Scotland", - FederationId("VEN") -> "Venezuela", - FederationId("IRL") -> "Ireland", - FederationId("URU") -> "Uruguay", - FederationId("TKM") -> "Turkmenistan", - FederationId("MAR") -> "Morocco", - FederationId("MAS") -> "Malaysia", - FederationId("BAN") -> "Bangladesh", - FederationId("ALG") -> "Algeria", - FederationId("RSA") -> "South Africa", - FederationId("AND") -> "Andorra", - FederationId("ALB") -> "Albania", - FederationId("KGZ") -> "Kyrgyzstan", - FederationId("KOS") -> "Kosovo *", - FederationId("FAI") -> "Faroe Islands", - FederationId("ZAM") -> "Zambia", - FederationId("MYA") -> "Myanmar", - FederationId("NZL") -> "New Zealand", - FederationId("ECU") -> "Ecuador", + FederationId("CHN") -> "China", + FederationId("CIV") -> "Cote d’Ivoire", + FederationId("CMR") -> "Cameroon", + FederationId("COD") -> "Democratic Republic of the Congo", + FederationId("COL") -> "Colombia", + FederationId("COM") -> "Comoros Islands", + FederationId("CPV") -> "Cape Verde", FederationId("CRC") -> "Costa Rica", - FederationId("NGR") -> "Nigeria", - FederationId("JPN") -> "Japan", - FederationId("SYR") -> "Syria", + FederationId("CRO") -> "Croatia", + FederationId("CUB") -> "Cuba", + FederationId("CYP") -> "Cyprus", + FederationId("CZE") -> "Czech Republic", + FederationId("DEN") -> "Denmark", + FederationId("DJI") -> "Djibouti", + FederationId("DMA") -> "Dominica", FederationId("DOM") -> "Dominican Republic", - FederationId("LUX") -> "Luxembourg", - FederationId("WLS") -> "Wales", - FederationId("BOL") -> "Bolivia", - FederationId("TUN") -> "Tunisia", - FederationId("UAE") -> "United Arab Emirates", - FederationId("MNC") -> "Monaco", - FederationId("TJK") -> "Tajikistan", - FederationId("PAN") -> "Panama", - FederationId("LBN") -> "Lebanon", - FederationId("NCA") -> "Nicaragua", + FederationId("ECU") -> "Ecuador", + FederationId("EGY") -> "Egypt", + FederationId("ENG") -> "England", + FederationId("ERI") -> "Eritrea", FederationId("ESA") -> "El Salvador", - FederationId("ANG") -> "Angola", - FederationId("TTO") -> "Trinidad & Tobago", - FederationId("SRI") -> "Sri Lanka", + FederationId("ESP") -> "Spain", + FederationId("EST") -> "Estonia", + FederationId("ETH") -> "Ethiopia", + FederationId("FAI") -> "Faroe Islands", + FederationId("FID") -> "FIDE", + FederationId("FIJ") -> "Fiji", + FederationId("FIN") -> "Finland", + FederationId("FRA") -> "France", + FederationId("GAB") -> "Gabon", + FederationId("GAM") -> "Gambia", + FederationId("GCI") -> "Guernsey", + FederationId("GEO") -> "Georgia", + FederationId("GEQ") -> "Equatorial Guinea", + FederationId("GER") -> "Germany", + FederationId("GHA") -> "Ghana", + FederationId("GRE") -> "Greece", + FederationId("GRL") -> "Greenland", + FederationId("GRN") -> "Grenada", + FederationId("GUA") -> "Guatemala", + FederationId("GUM") -> "Guam", + FederationId("GUY") -> "Guyana", + FederationId("HAI") -> "Haiti", + FederationId("HKG") -> "Hong Kong, China", + FederationId("HON") -> "Honduras", + FederationId("HUN") -> "Hungary", + FederationId("INA") -> "Indonesia", + FederationId("IND") -> "India", + FederationId("IOM") -> "Isle of Man", + FederationId("IRI") -> "Iran", + FederationId("IRL") -> "Ireland", FederationId("IRQ") -> "Iraq", + FederationId("ISL") -> "Iceland", + FederationId("ISR") -> "Israel", + FederationId("ISV") -> "US Virgin Islands", + FederationId("ITA") -> "Italy", + FederationId("IVB") -> "British Virgin Islands", + FederationId("JAM") -> "Jamaica", + FederationId("JCI") -> "Jersey", FederationId("JOR") -> "Jordan", - FederationId("UGA") -> "Uganda", - FederationId("MAD") -> "Madagascar", - FederationId("ZIM") -> "Zimbabwe", - FederationId("MLT") -> "Malta", - FederationId("SUD") -> "Sudan", + FederationId("JPN") -> "Japan", + FederationId("KAZ") -> "Kazakhstan", + FederationId("KEN") -> "Kenya", + FederationId("KGZ") -> "Kyrgyzstan", FederationId("KOR") -> "South Korea", - FederationId("PUR") -> "Puerto Rico", - FederationId("HON") -> "Honduras", - FederationId("GUA") -> "Guatemala", - FederationId("PAK") -> "Pakistan", - FederationId("JAM") -> "Jamaica", - FederationId("THA") -> "Thailand", - FederationId("YEM") -> "Yemen", + FederationId("KOS") -> "Kosovo *", + FederationId("KSA") -> "Saudi Arabia", + FederationId("KUW") -> "Kuwait", + FederationId("LAO") -> "Laos", + FederationId("LAT") -> "Latvia", FederationId("LBA") -> "Libya", - FederationId("CYP") -> "Cyprus", - FederationId("NEP") -> "Nepal", - FederationId("HKG") -> "Hong Kong, China", - FederationId("SSD") -> "South Sudan", - FederationId("BOT") -> "Botswana", - FederationId("PLE") -> "Palestine", - FederationId("KEN") -> "Kenya", - FederationId("AHO") -> "Netherlands Antilles", - FederationId("MAW") -> "Malawi", + FederationId("LBN") -> "Lebanon", + FederationId("LBR") -> "Liberia", + FederationId("LCA") -> "Saint Lucia", + FederationId("LES") -> "Lesotho", FederationId("LIE") -> "Liechtenstein", - FederationId("TPE") -> "Chinese Taipei", - FederationId("AFG") -> "Afghanistan", + FederationId("LTU") -> "Lithuania", + FederationId("LUX") -> "Luxembourg", + FederationId("MAC") -> "Macau", + FederationId("MAD") -> "Madagascar", + FederationId("MAR") -> "Morocco", + FederationId("MAS") -> "Malaysia", + FederationId("MAW") -> "Malawi", + FederationId("MDA") -> "Moldova", + FederationId("MDV") -> "Maldives", + FederationId("MEX") -> "Mexico", + FederationId("MGL") -> "Mongolia", + FederationId("MKD") -> "North Macedonia", + FederationId("MLI") -> "Mali", + FederationId("MLT") -> "Malta", + FederationId("MNC") -> "Monaco", + FederationId("MNE") -> "Montenegro", FederationId("MOZ") -> "Mozambique", - FederationId("KSA") -> "Saudi Arabia", - FederationId("BAR") -> "Barbados", - FederationId("NAM") -> "Namibia", - FederationId("HAI") -> "Haiti", - FederationId("ARU") -> "Aruba", - FederationId("CIV") -> "Cote d’Ivoire", - FederationId("CPV") -> "Cape Verde", - FederationId("SUR") -> "Suriname", - FederationId("LBR") -> "Liberia", - FederationId("IOM") -> "Isle of Man", - FederationId("MTN") -> "Mauritania", - FederationId("BRN") -> "Bahrain", - FederationId("GHA") -> "Ghana", - FederationId("OMA") -> "Oman", - FederationId("BRU") -> "Brunei Darussalam", - FederationId("GCI") -> "Guernsey", - FederationId("GUM") -> "Guam", - FederationId("KUW") -> "Kuwait", - FederationId("JCI") -> "Jersey", FederationId("MRI") -> "Mauritius", - FederationId("SEN") -> "Senegal", - FederationId("BAH") -> "Bahamas", - FederationId("MDV") -> "Maldives", + FederationId("MTN") -> "Mauritania", + FederationId("MYA") -> "Myanmar", + FederationId("NAM") -> "Namibia", + FederationId("NCA") -> "Nicaragua", + FederationId("NCL") -> "New Caledonia", + FederationId("NED") -> "Netherlands", + FederationId("NEP") -> "Nepal", + FederationId("NGR") -> "Nigeria", + FederationId("NIG") -> "Niger", + FederationId("NOR") -> "Norway", FederationId("NRU") -> "Nauru", - FederationId("TOG") -> "Togo", - FederationId("FIJ") -> "Fiji", + FederationId("NZL") -> "New Zealand", + FederationId("OMA") -> "Oman", + FederationId("PAK") -> "Pakistan", + FederationId("PAN") -> "Panama", + FederationId("PAR") -> "Paraguay", + FederationId("PER") -> "Peru", + FederationId("PHI") -> "Philippines", + FederationId("PLE") -> "Palestine", FederationId("PLW") -> "Palau", - FederationId("GUY") -> "Guyana", - FederationId("LES") -> "Lesotho", - FederationId("CAY") -> "Cayman Islands", - FederationId("SOM") -> "Somalia", - FederationId("SWZ") -> "Eswatini", - FederationId("TAN") -> "Tanzania", - FederationId("LCA") -> "Saint Lucia", - FederationId("ISV") -> "US Virgin Islands", - FederationId("SLE") -> "Sierra Leone", - FederationId("BER") -> "Bermuda", - FederationId("SMR") -> "San Marino", - FederationId("BDI") -> "Burundi", - FederationId("QAT") -> "Qatar", - FederationId("ETH") -> "Ethiopia", - FederationId("DJI") -> "Djibouti", - FederationId("SEY") -> "Seychelles", FederationId("PNG") -> "Papua New Guinea", - FederationId("DMA") -> "Dominica", - FederationId("STP") -> "Sao Tome and Principe", - FederationId("MAC") -> "Macau", - FederationId("CAM") -> "Cambodia", - FederationId("VIN") -> "Saint Vincent and the Grenadines", - FederationId("BUR") -> "Burkina Faso", - FederationId("COM") -> "Comoros Islands", - FederationId("GAB") -> "Gabon", + FederationId("POL") -> "Poland", + FederationId("POR") -> "Portugal", + FederationId("PUR") -> "Puerto Rico", + FederationId("QAT") -> "Qatar", + FederationId("ROU") -> "Romania", + FederationId("RSA") -> "South Africa", + FederationId("RUS") -> "Russia", FederationId("RWA") -> "Rwanda", - FederationId("CMR") -> "Cameroon", - FederationId("MLI") -> "Mali", - FederationId("ANT") -> "Antigua and Barbuda", - FederationId("CHA") -> "Chad", - FederationId("GAM") -> "Gambia", - FederationId("COD") -> "Democratic Republic of the Congo", + FederationId("SCO") -> "Scotland", + FederationId("SEN") -> "Senegal", + FederationId("SEY") -> "Seychelles", + FederationId("SGP") -> "Singapore", FederationId("SKN") -> "Saint Kitts and Nevis", - FederationId("BHU") -> "Bhutan", - FederationId("NIG") -> "Niger", - FederationId("GRN") -> "Grenada", - FederationId("BIZ") -> "Belize", - FederationId("CAF") -> "Central African Republic", - FederationId("ERI") -> "Eritrea", - FederationId("GEQ") -> "Equatorial Guinea", - FederationId("IVB") -> "British Virgin Islands", - FederationId("LAO") -> "Laos", + FederationId("SLE") -> "Sierra Leone", + FederationId("SLO") -> "Slovenia", + FederationId("SMR") -> "San Marino", FederationId("SOL") -> "Solomon Islands", + FederationId("SOM") -> "Somalia", + FederationId("SRB") -> "Serbia", + FederationId("SRI") -> "Sri Lanka", + FederationId("SSD") -> "South Sudan", + FederationId("STP") -> "Sao Tome and Principe", + FederationId("SUD") -> "Sudan", + FederationId("SUI") -> "Switzerland", + FederationId("SUR") -> "Suriname", + FederationId("SVK") -> "Slovakia", + FederationId("SWE") -> "Sweden", + FederationId("SWZ") -> "Eswatini", + FederationId("SYR") -> "Syria", + FederationId("TAN") -> "Tanzania", FederationId("TGA") -> "Tonga", + FederationId("THA") -> "Thailand", + FederationId("TJK") -> "Tajikistan", + FederationId("TKM") -> "Turkmenistan", FederationId("TLS") -> "Timor-Leste", - FederationId("VAN") -> "Vanuatu" + FederationId("TOG") -> "Togo", + FederationId("TPE") -> "Chinese Taipei", + FederationId("TTO") -> "Trinidad & Tobago", + FederationId("TUN") -> "Tunisia", + FederationId("TUR") -> "Turkiye", + FederationId("UAE") -> "United Arab Emirates", + FederationId("UGA") -> "Uganda", + FederationId("UKR") -> "Ukraine", + FederationId("URU") -> "Uruguay", + FederationId("USA") -> "United States of America", + FederationId("UZB") -> "Uzbekistan", + FederationId("VAN") -> "Vanuatu", + FederationId("VEN") -> "Venezuela", + FederationId("VIE") -> "Vietnam", + FederationId("VIN") -> "Saint Vincent and the Grenadines", + FederationId("WLS") -> "Wales", + FederationId("YEM") -> "Yemen", + FederationId("ZAM") -> "Zambia", + FederationId("ZIM") -> "Zimbabwe" )