diff --git a/csv-aggregator/pack.sh b/csv-aggregator/pack.sh index c3e35548..ffc5e848 100755 --- a/csv-aggregator/pack.sh +++ b/csv-aggregator/pack.sh @@ -43,6 +43,9 @@ cp merge.sql $tempDir echo "Merge files..." (cd $tempDir; clickhouse local --queries-file merge.sql) +ls -l $tempDir/result/raw.csv +ls -l $tempDir/result/deduplicated.csv + mv $tempDir/result/raw.csv "$tempDir/tw-rental-data/${targetName}-raw.csv" (cd $tempDir; zip -r "[$targetName][CSV][Raw] TW-Rental-Data.zip" tw-rental-data; mv tw-rental-data/*.csv ./result) diff --git a/ui/assets/stats/2020.json b/ui/assets/stats/2020.json index 2e1f085f..ef86a4ed 100644 --- a/ui/assets/stats/2020.json +++ b/ui/assets/stats/2020.json @@ -10,8 +10,8 @@ {"name": "591", "count": 1285681} ], "files": [ - {"format": "csv", "size_byte": 454731213, "download_url": "http://bit.ly/tw-rental-data-2020-raw-csv"}, - {"format": "json", "size_byte": 2660914792, "download_url": "http://bit.ly/tw-rental-data-2020-raw-json"} + {"format": "csv", "size_byte": 454731213, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 2660914792, "download_url": { "isS3": true }} ], "comment": "" }, @@ -25,8 +25,8 @@ {"name": "591", "count": 826718} ], "files": [ - {"format": "csv", "size_byte": 284934216, "download_url": "http://bit.ly/tw-rental-data-2020-dedup-csv"}, - {"format": "json", "size_byte": 1708650356, "download_url": "http://bit.ly/tw-rental-data-2020-dedup-json"} + {"format": "csv", "size_byte": 284934216, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 1708650356, "download_url": { "isS3": true }} ], "comment": "" } @@ -42,8 +42,8 @@ {"name": "591", "count": 334876} ], "files": [ - {"format": "csv", "size_byte": 126456849, "download_url": "http://bit.ly/tw-rental-data-2020-Q1-raw-csv"}, - {"format": "json", "size_byte": 692445549, "download_url": "http://bit.ly/tw-rental-data-2020-Q1-raw-json"} + {"format": "csv", "size_byte": 126456849, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 692445549, "download_url": { "isS3": true }} ] }, { @@ -56,8 +56,8 @@ {"name": "591", "count": 238680} ], "files": [ - {"format": "csv", "size_byte": 352865876, "download_url": "http://bit.ly/tw-rental-data-2020-Q1-dedup-csv"}, - {"format": "json", "size_byte": 492679283, "download_url": "http://bit.ly/tw-rental-data-2020-Q1-dedup-json"} + {"format": "csv", "size_byte": 352865876, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 492679283, "download_url": { "isS3": true }} ] }, { @@ -70,8 +70,8 @@ {"name": "591", "count": 385231} ], "files": [ - {"format": "csv", "size_byte": 135597419, "download_url": "http://bit.ly/tw-rental-data-2020-Q2-raw-csv"}, - {"format": "json", "size_byte": 796677029, "download_url": "http://bit.ly/tw-rental-data-2020-Q2-raw-json"} + {"format": "csv", "size_byte": 135597419, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 796677029, "download_url": { "isS3": true }} ] }, { @@ -84,8 +84,8 @@ {"name": "591", "count": 269997} ], "files": [ - {"format": "csv", "size_byte": 92350551, "download_url": "http://bit.ly/tw-rental-data-2020-Q2-dedup-csv"}, - {"format": "json", "size_byte": 557429946, "download_url": "http://bit.ly/tw-rental-data-2020-Q2-dedup-json"} + {"format": "csv", "size_byte": 92350551, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 557429946, "download_url": { "isS3": true }} ] }, { @@ -98,8 +98,8 @@ {"name": "591", "count": 380161} ], "files": [ - {"format": "csv", "size_byte": 134008644, "download_url": "http://bit.ly/tw-rental-data-2020-Q3-raw-csv"}, - {"format": "json", "size_byte": 786560494, "download_url": "http://bit.ly/tw-rental-data-2020-Q3-raw-json"} + {"format": "csv", "size_byte": 134008644, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 786560494, "download_url": { "isS3": true }} ] }, { @@ -112,8 +112,8 @@ {"name": "591", "count": 271188} ], "files": [ - {"format": "csv", "size_byte": 92886480, "download_url": "http://bit.ly/tw-rental-data-2020-Q3-dedup-csv"}, - {"format": "json", "size_byte": 560118715, "download_url": "http://bit.ly/tw-rental-data-2020-Q3-dedup-json"} + {"format": "csv", "size_byte": 92886480, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 560118715, "download_url": { "isS3": true }} ] }, { @@ -126,8 +126,8 @@ {"name": "591", "count": 366093} ], "files": [ - {"format": "csv", "size_byte": 129875785, "download_url": "http://bit.ly/tw-rental-data-2020-Q4-raw-csv"}, - {"format": "json", "size_byte": 758104222, "download_url": "http://bit.ly/tw-rental-data-2020-Q4-raw-json"} + {"format": "csv", "size_byte": 129875785, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 758104222, "download_url": { "isS3": true }} ] }, { @@ -140,8 +140,8 @@ {"name": "591", "count": 257815} ], "files": [ - {"format": "csv", "size_byte": 88936239, "download_url": "http://bit.ly/tw-rental-data-2020-Q4-dedup-csv"}, - {"format": "json", "size_byte": 532989653, "download_url": "http://bit.ly/tw-rental-data-2020-Q4-dedup-json"} + {"format": "csv", "size_byte": 88936239, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 532989653, "download_url": { "isS3": true }} ] } ], @@ -156,8 +156,8 @@ {"name": "591", "count": 129114} ], "files": [ - {"format": "csv", "size_byte": 45157668, "download_url": "http://bit.ly/tw-rental-data-2020-01-raw-csv"}, - {"format": "json", "size_byte": 266785175, "download_url": "http://bit.ly/tw-rental-data-2020-01-raw-json"} + {"format": "csv", "size_byte": 45157668, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 266785175, "download_url": { "isS3": true }} ], "comment": "" }, @@ -171,8 +171,8 @@ {"name": "591", "count": 100425} ], "files": [ - {"format": "csv", "size_byte": 34079504, "download_url": "http://bit.ly/tw-rental-data-2020-01-dedup-csv"}, - {"format": "json", "size_byte": 207105796, "download_url": "http://bit.ly/tw-rental-data-2020-01-dedup-json"} + {"format": "csv", "size_byte": 34079504, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 207105796, "download_url": { "isS3": true }} ], "comment": "" }, @@ -186,8 +186,8 @@ {"name": "591", "count": 143264} ], "files": [ - {"format": "csv", "size_byte": 50168802, "download_url": "http://bit.ly/tw-rental-data-2020-02-raw-csv"}, - {"format": "json", "size_byte": 296090982, "download_url": "http://bit.ly/tw-rental-data-2020-02-raw-json"} + {"format": "csv", "size_byte": 50168802, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 296090982, "download_url": { "isS3": true }} ], "comment": "" }, @@ -201,8 +201,8 @@ {"name": "591", "count": 115642} ], "files": [ - {"format": "csv", "size_byte": 39316873, "download_url": "http://bit.ly/tw-rental-data-2020-02-dedup-csv"}, - {"format": "json", "size_byte": 238556133, "download_url": "http://bit.ly/tw-rental-data-2020-02-dedup-json"} + {"format": "csv", "size_byte": 39316873, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 238556133, "download_url": { "isS3": true }} ], "comment": "" }, @@ -216,8 +216,8 @@ {"name": "591", "count": 170735} ], "files": [ - {"format": "csv", "size_byte": 59798293, "download_url": "http://bit.ly/tw-rental-data-2020-03-raw-csv"}, - {"format": "json", "size_byte": 352865876, "download_url": "http://bit.ly/tw-rental-data-2020-03-raw-json"} + {"format": "csv", "size_byte": 59798293, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 352865876, "download_url": { "isS3": true }} ], "comment": "" }, @@ -231,8 +231,8 @@ {"name": "591", "count": 133329} ], "files": [ - {"format": "csv", "size_byte": 45314326, "download_url": "http://bit.ly/tw-rental-data-2020-03-dedup-csv"}, - {"format": "json", "size_byte": 275028324, "download_url": "http://bit.ly/tw-rental-data-2020-03-dedup-json"} + {"format": "csv", "size_byte": 45314326, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 275028324, "download_url": { "isS3": true }} ], "comment": "" }, @@ -246,8 +246,8 @@ {"name": "591", "count": 169102} ], "files": [ - {"format": "csv", "size_byte": 59218473, "download_url": "http://bit.ly/tw-rental-data-2020-04-raw-csv"}, - {"format": "json", "size_byte": 349450489, "download_url": "http://bit.ly/tw-rental-data-2020-04-raw-json"} + {"format": "csv", "size_byte": 59218473, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 349450489, "download_url": { "isS3": true }} ], "comment": "" }, @@ -261,8 +261,8 @@ {"name": "591", "count": 131845} ], "files": [ - {"format": "csv", "size_byte": 44803192, "download_url": "http://bit.ly/tw-rental-data-2020-04-dedup-csv"}, - {"format": "json", "size_byte": 271940497, "download_url": "http://bit.ly/tw-rental-data-2020-04-dedup-json"} + {"format": "csv", "size_byte": 44803192, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 271940497, "download_url": { "isS3": true }} ], "comment": "" }, @@ -276,8 +276,8 @@ {"name": "591", "count": 174726} ], "files": [ - {"format": "csv", "size_byte": 61204437, "download_url": "http://bit.ly/tw-rental-data-2020-05-raw-csv"}, - {"format": "json", "size_byte": 361104156, "download_url": "http://bit.ly/tw-rental-data-2020-05-raw-json"} + {"format": "csv", "size_byte": 61204437, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 361104156, "download_url": { "isS3": true }} ], "comment": "" }, @@ -291,8 +291,8 @@ {"name": "591", "count": 134563} ], "files": [ - {"format": "csv", "size_byte": 45745320, "download_url": "http://bit.ly/tw-rental-data-2020-05-dedup-csv"}, - {"format": "json", "size_byte": 277586686, "download_url": "http://bit.ly/tw-rental-data-2020-05-dedup-json"} + {"format": "csv", "size_byte": 45745320, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 277586686, "download_url": { "isS3": true }} ], "comment": "" }, @@ -306,8 +306,8 @@ {"name": "591", "count": 168115} ], "files": [ - {"format": "csv", "size_byte": 58956597, "download_url": "http://bit.ly/tw-rental-data-2020-06-raw-csv"}, - {"format": "json", "size_byte": 347560217, "download_url": "http://bit.ly/tw-rental-data-2020-06-raw-json"} + {"format": "csv", "size_byte": 58956597, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 347560217, "download_url": { "isS3": true }} ], "comment": "" }, @@ -321,8 +321,8 @@ {"name": "591", "count": 131907} ], "files": [ - {"format": "csv", "size_byte": 44882562, "download_url": "http://bit.ly/tw-rental-data-2020-06-dedup-csv"}, - {"format": "json", "size_byte": 272186079, "download_url": "http://bit.ly/tw-rental-data-2020-06-dedup-json"} + {"format": "csv", "size_byte": 44882562, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 272186079, "download_url": { "isS3": true }} ], "comment": "" }, @@ -336,8 +336,8 @@ {"name": "591", "count": 170543} ], "files": [ - {"format": "csv", "size_byte": 59808733, "download_url": "http://bit.ly/tw-rental-data-2020-07-raw-csv"}, - {"format": "json", "size_byte": 352609352, "download_url": "http://bit.ly/tw-rental-data-2020-07-raw-json"} + {"format": "csv", "size_byte": 59808733, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 352609352, "download_url": { "isS3": true }} ], "comment": "" }, @@ -351,8 +351,8 @@ {"name": "591", "count": 134329} ], "files": [ - {"format": "csv", "size_byte": 45710914, "download_url": "http://bit.ly/tw-rental-data-2020-07-dedup-csv"}, - {"format": "json", "size_byte": 277201480, "download_url": "http://bit.ly/tw-rental-data-2020-07-dedup-json"} + {"format": "csv", "size_byte": 45710914, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 277201480, "download_url": { "isS3": true }} ], "comment": "" }, @@ -366,8 +366,8 @@ {"name": "591", "count": 168450} ], "files": [ - {"format": "csv", "size_byte": 348305241, "download_url": "http://bit.ly/tw-rental-data-2020-08-raw-csv"}, - {"format": "json", "size_byte": 348305241, "download_url": "http://bit.ly/tw-rental-data-2020-08-raw-json"} + {"format": "csv", "size_byte": 348305241, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 348305241, "download_url": { "isS3": true }} ], "comment": "" }, @@ -381,8 +381,8 @@ {"name": "591", "count": 132469} ], "files": [ - {"format": "csv", "size_byte": 45064826, "download_url": "http://bit.ly/tw-rental-data-2020-08-dedup-csv"}, - {"format": "json", "size_byte": 273366338, "download_url": "http://bit.ly/tw-rental-data-2020-08-dedup-json"} + {"format": "csv", "size_byte": 45064826, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 273366338, "download_url": { "isS3": true }} ], "comment": "" }, @@ -396,8 +396,8 @@ {"name": "591", "count": 161185} ], "files": [ - {"format": "csv", "size_byte": 56602782, "download_url": "http://bit.ly/tw-rental-data-2020-09-raw-csv"}, - {"format": "json", "size_byte": 333324801, "download_url": "http://bit.ly/tw-rental-data-2020-09-raw-json"} + {"format": "csv", "size_byte": 56602782, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 333324801, "download_url": { "isS3": true }} ], "comment": "" }, @@ -411,8 +411,8 @@ {"name": "591", "count": 125815} ], "files": [ - {"format": "csv", "size_byte": 42860375, "download_url": "http://bit.ly/tw-rental-data-2020-09-dedup-csv"}, - {"format": "json", "size_byte": 259676937, "download_url": "http://bit.ly/tw-rental-data-2020-09-dedup-json"} + {"format": "csv", "size_byte": 42860375, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 259676937, "download_url": { "isS3": true }} ], "comment": "" }, @@ -426,8 +426,8 @@ {"name": "591", "count": 162858} ], "files": [ - {"format": "csv", "size_byte": 57314984, "download_url": "http://bit.ly/tw-rental-data-2020-10-raw-csv"}, - {"format": "json", "size_byte": 336866283, "download_url": "http://bit.ly/tw-rental-data-2020-10-raw-json"} + {"format": "csv", "size_byte": 57314984, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 336866283, "download_url": { "isS3": true }} ], "comment": "" }, @@ -441,8 +441,8 @@ {"name": "591", "count": 126773} ], "files": [ - {"format": "csv", "size_byte": 43295178, "download_url": "http://bit.ly/tw-rental-data-2020-10-dedup-csv"}, - {"format": "json", "size_byte": 261719952, "download_url": "http://bit.ly/tw-rental-data-2020-10-dedup-json"} + {"format": "csv", "size_byte": 43295178, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 261719952, "download_url": { "isS3": true }} ], "comment": "" }, @@ -456,8 +456,8 @@ {"name": "591", "count": 159694} ], "files": [ - {"format": "csv", "size_byte": 56429557, "download_url": "http://bit.ly/tw-rental-data-2020-11-raw-csv"}, - {"format": "json", "size_byte": 330535305, "download_url": "http://bit.ly/tw-rental-data-2020-11-raw-json"} + {"format": "csv", "size_byte": 56429557, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 330535305, "download_url": { "isS3": true }} ], "comment": "" }, @@ -471,8 +471,8 @@ {"name": "591", "count": 123428} ], "files": [ - {"format": "csv", "size_byte": 42333912, "download_url": "http://bit.ly/tw-rental-data-2020-11-dedup-csv"}, - {"format": "json", "size_byte": 254986016, "download_url": "http://bit.ly/tw-rental-data-2020-11-dedup-json"} + {"format": "csv", "size_byte": 42333912, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 254986016, "download_url": { "isS3": true }} ], "comment": "" }, @@ -486,8 +486,8 @@ {"name": "591", "count": 160948} ], "files": [ - {"format": "csv", "size_byte": 57007509, "download_url": "http://bit.ly/tw-rental-data-2020-12-raw-csv"}, - {"format": "json", "size_byte": 333255381, "download_url": "http://bit.ly/tw-rental-data-2020-12-raw-json"} + {"format": "csv", "size_byte": 57007509, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 333255381, "download_url": { "isS3": true }} ], "comment": "" }, @@ -501,8 +501,8 @@ {"name": "591", "count": 124471} ], "files": [ - {"format": "csv", "size_byte": 42824330, "download_url": "http://bit.ly/tw-rental-data-2020-12-dedup-csv"}, - {"format": "json", "size_byte": 257253241, "download_url": "http://bit.ly/tw-rental-data-2020-12-dedup-json"} + {"format": "csv", "size_byte": 42824330, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 257253241, "download_url": { "isS3": true }} ], "comment": "" } diff --git a/ui/assets/stats/2021.json b/ui/assets/stats/2021.json index 0a65f929..d3303dc2 100644 --- a/ui/assets/stats/2021.json +++ b/ui/assets/stats/2021.json @@ -14,7 +14,10 @@ {"format": "csv", "size_byte": 410321284, "download_url": { "isS3": true }}, {"format": "json", "size_byte": 2378275777, "download_url": { "isS3": true }} ], - "comment": "" + "comment": [ + "爬蟲在 2 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/tag/%E8%B3%87%E6%96%99%E9%8C%AF%E8%AA%A4/)", + "爬蟲在 6/17 - 10/30 期間沒有完整執行,估計遺失 39% 的物件,詳情[請見影響評估](/blog/post/data-issue-2021-01)" + ] }, { "schema_ver": "1.0.0", @@ -29,7 +32,10 @@ {"format": "csv", "size_byte": 285385766, "download_url": { "isS3": true }}, {"format": "json", "size_byte": 1700724070, "download_url": { "isS3": true }} ], - "comment": "" + "comment": [ + "爬蟲在 2 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/tag/%E8%B3%87%E6%96%99%E9%8C%AF%E8%AA%A4/)", + "爬蟲在 6/17 - 10/30 期間沒有完整執行,估計遺失 39% 的物件,詳情[請見影響評估](/blog/post/data-issue-2021-01)" + ] } ], "quarterly": [ @@ -43,8 +49,8 @@ {"name": "591", "count": 343773} ], "files": [ - {"format": "csv", "size_byte": 122163781, "download_url": "http://bit.ly/tw-rental-data-2021-Q1-raw-csv"}, - {"format": "json", "size_byte": 712151409, "download_url": "http://bit.ly/tw-rental-data-2021-Q1-raw-json"} + {"format": "csv", "size_byte": 122163781, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 712151409, "download_url": { "isS3": true }} ], "comment": "爬蟲在 2 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/tag/%E8%B3%87%E6%96%99%E9%8C%AF%E8%AA%A4/)" }, @@ -58,8 +64,8 @@ {"name": "591", "count": 246232} ], "files": [ - {"format": "csv", "size_byte": 85158890, "download_url": "http://bit.ly/tw-rental-data-2021-Q1-dedup-csv"}, - {"format": "json", "size_byte": 509272899, "download_url": "http://bit.ly/tw-rental-data-2021-Q1-dedup-json"} + {"format": "csv", "size_byte": 85158890, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 509272899, "download_url": { "isS3": true }} ], "comment": "爬蟲在 2 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/tag/%E8%B3%87%E6%96%99%E9%8C%AF%E8%AA%A4/)" }, @@ -165,8 +171,8 @@ {"name": "591", "count": 157416} ], "files": [ - {"format": "csv", "size_byte": 55776559, "download_url": "http://bit.ly/tw-rentaldata-2021-01-raw-csv"}, - {"format": "json", "size_byte": 325953131, "download_url": "http://bit.ly/tw-rental-data-2021-01-raw-json"} + {"format": "csv", "size_byte": 55776559, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 325953131, "download_url": { "isS3": true }} ], "comment": "" }, @@ -180,8 +186,8 @@ {"name": "591", "count": 122008} ], "files": [ - {"format": "csv", "size_byte": 41982752, "download_url": "http://bit.ly/tw-rentaldata-2021-01-dedup-csv"}, - {"format": "json", "size_byte": 252163294, "download_url": "http://bit.ly/tw-rental-data-2021-01-dedup-json"} + {"format": "csv", "size_byte": 41982752, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 252163294, "download_url": { "isS3": true }} ], "comment": "" }, @@ -195,8 +201,8 @@ {"name": "591", "count": 137367} ], "files": [ - {"format": "csv", "size_byte": 48496230, "download_url": "http://bit.ly/tw-rental-data-2021-02-raw-csv"}, - {"format": "json", "size_byte": 284314508, "download_url": "http://bit.ly/tw-rental-data-2021-02-raw-json"} + {"format": "csv", "size_byte": 48496230, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 284314508, "download_url": { "isS3": true }} ], "comment": "爬蟲在 [2/20 - 2/23 期間沒有完整執行](/blog/post/data-issue-2021-00),只在這 4 天出現的物件可能會消失,其他物件在這段期間的新增、刪除、修改,也可能會有時間登記錯誤的問題。" }, @@ -210,8 +216,8 @@ {"name": "591", "count": 108074} ], "files": [ - {"format": "csv", "size_byte": 37063844, "download_url": "http://bit.ly/tw-rental-data-2021-02-dedup-csv"}, - {"format": "json", "size_byte": 223276613, "download_url": "http://bit.ly/tw-rental-data-2021-02-dedup-json"} + {"format": "csv", "size_byte": 37063844, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 223276613, "download_url": { "isS3": true }} ], "comment": "爬蟲在 [2/20 - 2/23 期間沒有完整執行](/blog/post/data-issue-2021-00),只在這 4 天出現的物件可能會消失,其他物件在這段期間的新增、刪除、修改,也可能會有時間登記錯誤的問題。" }, @@ -225,8 +231,8 @@ {"name": "591", "count": 161679} ], "files": [ - {"format": "csv", "size_byte": 57250461, "download_url": "http://bit.ly/tw-rental-data-2021-03-raw-csv"}, - {"format": "json", "size_byte": 334796242, "download_url": "http://bit.ly/tw-rental-data-2021-03-raw-json"} + {"format": "csv", "size_byte": 57250461, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 334796242, "download_url": { "isS3": true }} ], "comment": "" }, @@ -240,8 +246,8 @@ {"name": "591", "count": 129362} ], "files": [ - {"format": "csv", "size_byte": 44532070, "download_url": "http://bit.ly/tw-rental-data-2021-03-dedup-csv"}, - {"format": "json", "size_byte": 267407088, "download_url": "http://bit.ly/tw-rental-data-2021-03-dedup-json"} + {"format": "csv", "size_byte": 44532070, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 267407088, "download_url": { "isS3": true }} ], "comment": "" }, @@ -255,8 +261,8 @@ {"name": "591", "count": 159624} ], "files": [ - {"format": "csv", "size_byte": 56501119, "download_url": "http://bit.ly/tw-rental-data-2021-04-raw-csv"}, - {"format": "json", "size_byte": 330544392, "download_url": "http://bit.ly/tw-rental-data-2021-04-raw-json"} + {"format": "csv", "size_byte": 56501119, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 330544392, "download_url": { "isS3": true }} ], "comment": "" }, @@ -270,8 +276,8 @@ {"name": "591", "count": 126265} ], "files": [ - {"format": "csv", "size_byte": 43418967, "download_url": "http://bit.ly/tw-rental-data-2021-04-dedup-csv"}, - {"format": "json", "size_byte": 260985441, "download_url": "http://bit.ly/tw-rental-data-2021-04-dedup-json"} + {"format": "csv", "size_byte": 43418967, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 260985441, "download_url": { "isS3": true }} ], "comment": "" }, @@ -285,8 +291,8 @@ {"name": "591", "count": 148521} ], "files": [ - {"format": "csv", "size_byte": 52604899, "download_url": "http://bit.ly/tw-rental-data-2021-05-raw-csv"}, - {"format": "json", "size_byte": 307625181, "download_url": "http://bit.ly/tw-rental-data-2021-05-raw-json"} + {"format": "csv", "size_byte": 52604899, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 307625181, "download_url": { "isS3": true }} ], "comment": "" }, @@ -300,8 +306,8 @@ {"name": "591", "count": 117272} ], "files": [ - {"format": "csv", "size_byte": 40349266, "download_url": "http://bit.ly/tw-rental-data-2021-05-dedup-csv"}, - {"format": "json", "size_byte": 242449922, "download_url": "http://bit.ly/tw-rental-data-2021-05-dedup-json"} + {"format": "csv", "size_byte": 40349266, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 242449922, "download_url": { "isS3": true }} ], "comment": "" }, diff --git a/ui/assets/stats/2023.json b/ui/assets/stats/2023.json index c588fb0d..f6bb18cd 100644 --- a/ui/assets/stats/2023.json +++ b/ui/assets/stats/2023.json @@ -1,6 +1,40 @@ { "year": 2023, "annual": [ + { + "schema_ver": "1.0.0", + "data_ver": "0.3", + "time": "1", + "type": "原始資料", + "total_count": 1501986, + "sources": [ + {"name": "591", "count": 1501986} + ], + "files": [ + {"format": "csv", "size_byte": 654302780, "download_url": { "isS3": true }} + ], + "comment": [ + "爬蟲在 3-4 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/post/data-issue-2023-00)", + "爬蟲在 10 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/post/data-issue-2023-01)" + ] + }, + { + "schema_ver": "1.0.0", + "data_ver": "0.3", + "time": "1", + "type": "消除重複住宅", + "total_count": 1003599, + "sources": [ + {"name": "591", "count": 1003599} + ], + "files": [ + {"format": "csv", "size_byte": 416900306, "download_url": { "isS3": true }} + ], + "comment": [ + "爬蟲在 3-4 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/post/data-issue-2023-00)", + "爬蟲在 10 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/post/data-issue-2023-01)" + ] + } ], "quarterly": [ { @@ -86,6 +120,34 @@ {"format": "csv", "size_byte": 131518367, "download_url": { "isS3": true }} ], "comment": "" + }, + { + "schema_ver": "1.0.0", + "data_ver": "0.3", + "time": "4", + "type": "原始資料", + "total_count": 427514, + "sources": [ + {"name": "591", "count": 427514} + ], + "files": [ + {"format": "csv", "size_byte": 185966599, "download_url": { "isS3": true }} + ], + "comment": "爬蟲在 10 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/post/data-issue-2023-01)" + }, + { + "schema_ver": "1.0.0", + "data_ver": "0.3", + "time": "4", + "type": "消除重複住宅", + "total_count": 322542, + "sources": [ + {"name": "591", "count": 322542} + ], + "files": [ + {"format": "csv", "size_byte": 133753230, "download_url": { "isS3": true }} + ], + "comment": "爬蟲在 10 月曾出現資料錯誤,將會影響部份物件的新增與更新日期,詳細資訊請參見[部落格](/blog/post/data-issue-2023-01)" } ], "monthly": [ @@ -418,6 +480,36 @@ {"format": "json", "size_byte": 319232964, "download_url": { "isS3": true }} ], "comment": "" + }, + { + "schema_ver": "1.0.0", + "data_ver": "0.2", + "time": "12", + "type": "原始資料", + "total_count": 197126, + "sources": [ + {"name": "591", "count": 197126} + ], + "files": [ + {"format": "csv", "size_byte": 74175207, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 412202367, "download_url": { "isS3": true }} + ], + "comment": "" + }, + { + "schema_ver": "1.0.0", + "data_ver": "0.2", + "time": "12", + "type": "消除重複住宅", + "total_count": 151950, + "sources": [ + {"name": "591", "count": 151950} + ], + "files": [ + {"format": "csv", "size_byte": 54785172, "download_url": { "isS3": true }}, + {"format": "json", "size_byte": 316207681, "download_url": { "isS3": true }} + ], + "comment": "" } ] } diff --git a/ui/libs/defs.js b/ui/libs/defs.js index 8494ff3e..04173494 100644 --- a/ui/libs/defs.js +++ b/ui/libs/defs.js @@ -1 +1 @@ -export const S3_BASE = 'https://tw-rental-data.s3.us-west-2.amazonaws.com/' +export const S3_BASE = 'https://twrh.s3.ap-northeast-3.amazonaws.com/'