-
Notifications
You must be signed in to change notification settings - Fork 862
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Classify search engine results page content
- Loading branch information
Showing
29 changed files
with
1,085 additions
and
216 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
60 changes: 60 additions & 0 deletions
60
vendor/bat-native-ads/src/bat/ads/internal/base/search_engine_domain_extensions_util.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| /* Copyright (c) 2022 The Brave Authors. All rights reserved. | ||
| * This Source Code Form is subject to the terms of the Mozilla Public | ||
| * License, v. 2.0. If a copy of the MPL was not distributed with this file, | ||
| * You can obtain one at http://mozilla.org/MPL/2.0/. */ | ||
|
|
||
| #include "bat/ads/internal/base/search_engine_domain_extensions_util.h" | ||
|
|
||
| #include "base/no_destructor.h" | ||
|
|
||
| namespace ads { | ||
|
|
||
| const std::vector<std::string>& GetAmazonSearchEngineDomainExtensions() { | ||
| // See https://www.amazon.com/gp/navigation-country/select-country. | ||
| static base::NoDestructor<std::vector<std::string>> extensions( | ||
| {"ae", "ca", "cn", "co.jp", "co.uk", "com", "com.au", | ||
| "com.br", "com.mx", "de", "eg", "es", "fr", "in", | ||
| "it", "nl", "pl", "sa", "se", "sp", "tr"}); | ||
| return *extensions; | ||
| } | ||
|
|
||
| const std::vector<std::string>& GetGoogleSearchEngineDomainExtensions() { | ||
| static base::NoDestructor<std::vector<std::string>> extensions( | ||
| {"ac", "ad", "ae", "al", "am", "as", "at", | ||
| "az", "ba", "be", "bf", "bg", "bi", "bj", | ||
| "bs", "bt", "ca", "cat", "cd", "cf", "cg", | ||
| "ch", "ci", "ci", "cl", "cm", "cn", "co.bw", | ||
| "co.ck", "co.cr", "co.id", "co.il", "co.im", "co.in", "co.je", | ||
| "co.jp", "co.ke", "co.kr", "co.ls", "co.ma", "co.mz", "co.nz", | ||
| "co.th", "co.tz", "co.ug", "co.uk", "co.uz", "co.ve", "co.vi", | ||
| "co.za", "co.zm", "co.zw", "com.af", "com.ag", "com.ai", "com.ar", | ||
| "com.au", "com.bd", "com.bh", "com.bn", "com.bo", "com.br", "com.by", | ||
| "com.bz", "com.co", "com.cu", "com.cy", "com.do", "com.ec", "com.eg", | ||
| "com.et", "com.fj", "com.gh", "com.gi", "com.gt", "com.hk", "com.jm", | ||
| "com.kg", "com.kh", "com.kw", "com.lb", "com.ly", "com.mt", "com.mx", | ||
| "com.my", "com.na", "com.nf", "com.ng", "com.ni", "com.np", "com.om", | ||
| "com.pa", "com.pe", "com.pg", "com.ph", "com.pk", "com.pr", "com.py", | ||
| "com.qa", "com.sa", "com.sb", "com.sg", "com.sl", "com.sv", "com.tj", | ||
| "com.tr", "com.tw", "com.ua", "com.uy", "com.vc", "com.vn", "com", | ||
| "cv", "cz", "de", "dj", "dk", "dm", "dz", | ||
| "ee", "es", "fi", "fm", "fr", "ga", "ge", | ||
| "gg", "gl", "gm", "gp", "gr", "gy", "hn", | ||
| "hr", "ht", "hu", "ie", "iq", "is", "it.ao", | ||
| "it", "jo", "ki", "kz", "la", "li", "lk", | ||
| "lt", "lu", "lv", "md", "me", "mg", "mk", | ||
| "ml", "mn", "ms", "mu", "mv", "mw", "ne", | ||
| "nl", "no", "nr", "nu", "pl", "pn", "ps", | ||
| "pt", "ro", "rs", "ru", "rw", "sc", "se", | ||
| "sh", "si", "sk", "sm", "sn", "so", "sr", | ||
| "st", "td", "tg", "tk", "tl", "tm", "tn", | ||
| "to", "tt", "vg", "vu", "ws"}); | ||
| return *extensions; | ||
| } | ||
|
|
||
| const std::vector<std::string>& GetMojeekSearchEngineDomainExtensions() { | ||
| static base::NoDestructor<std::vector<std::string>> extensions( | ||
| {"co.uk", "com"}); | ||
| return *extensions; | ||
| } | ||
|
|
||
| } // namespace ads |
20 changes: 20 additions & 0 deletions
20
vendor/bat-native-ads/src/bat/ads/internal/base/search_engine_domain_extensions_util.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| /* Copyright (c) 2022 The Brave Authors. All rights reserved. | ||
| * This Source Code Form is subject to the terms of the Mozilla Public | ||
| * License, v. 2.0. If a copy of the MPL was not distributed with this file, | ||
| * You can obtain one at http://mozilla.org/MPL/2.0/. */ | ||
|
|
||
| #ifndef BRAVE_VENDOR_BAT_NATIVE_ADS_SRC_BAT_ADS_INTERNAL_BASE_SEARCH_ENGINE_DOMAIN_EXTENSIONS_UTIL_H_ | ||
| #define BRAVE_VENDOR_BAT_NATIVE_ADS_SRC_BAT_ADS_INTERNAL_BASE_SEARCH_ENGINE_DOMAIN_EXTENSIONS_UTIL_H_ | ||
|
|
||
| #include <string> | ||
| #include <vector> | ||
|
|
||
| namespace ads { | ||
|
|
||
| const std::vector<std::string>& GetAmazonSearchEngineDomainExtensions(); | ||
| const std::vector<std::string>& GetGoogleSearchEngineDomainExtensions(); | ||
| const std::vector<std::string>& GetMojeekSearchEngineDomainExtensions(); | ||
|
|
||
| } // namespace ads | ||
|
|
||
| #endif // BRAVE_VENDOR_BAT_NATIVE_ADS_SRC_BAT_ADS_INTERNAL_BASE_SEARCH_ENGINE_DOMAIN_EXTENSIONS_UTIL_H_ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
44 changes: 44 additions & 0 deletions
44
...r/bat-native-ads/src/bat/ads/internal/base/search_engine_results_page_url_pattern_util.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| /* Copyright (c) 2022 The Brave Authors. All rights reserved. | ||
| * This Source Code Form is subject to the terms of the Mozilla Public | ||
| * License, v. 2.0. If a copy of the MPL was not distributed with this file, | ||
| * You can obtain one at http://mozilla.org/MPL/2.0/. */ | ||
|
|
||
| #include "bat/ads/internal/base/search_engine_results_page_url_pattern_util.h" | ||
|
|
||
| #include "base/no_destructor.h" | ||
| #include "base/strings/strcat.h" | ||
| #include "bat/ads/internal/base/search_engine_url_pattern_util.h" | ||
|
|
||
| namespace ads { | ||
|
|
||
| const std::string& GetAmazonResultsPageUrlPattern() { | ||
| static base::NoDestructor<std::string> url_pattern( | ||
| base::StrCat({GetAmazonUrlPattern(), "s"})); | ||
| return *url_pattern; | ||
| } | ||
|
|
||
| const std::string& GetGoogleResultsPageUrlPattern() { | ||
| static base::NoDestructor<std::string> url_pattern( | ||
| base::StrCat({GetGoogleUrlPattern(), "search"})); | ||
| return *url_pattern; | ||
| } | ||
|
|
||
| const std::string& GetMojeekResultsPageUrlPattern() { | ||
| static base::NoDestructor<std::string> url_pattern( | ||
| base::StrCat({GetMojeekUrlPattern(), "search"})); | ||
| return *url_pattern; | ||
| } | ||
|
|
||
| const std::string& GetWikipediaResultsPageUrlPattern() { | ||
| static base::NoDestructor<std::string> url_pattern( | ||
| base::StrCat({GetWikipediaUrlPattern(), "wiki/(.*)"})); | ||
| return *url_pattern; | ||
| } | ||
|
|
||
| const std::string& GetYahooResultsPageUrlPattern() { | ||
| static base::NoDestructor<std::string> url_pattern( | ||
| base::StrCat({GetYahooUrlPattern(), "search(.*)"})); | ||
| return *url_pattern; | ||
| } | ||
|
|
||
| } // namespace ads |
21 changes: 21 additions & 0 deletions
21
...or/bat-native-ads/src/bat/ads/internal/base/search_engine_results_page_url_pattern_util.h
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| /* Copyright (c) 2022 The Brave Authors. All rights reserved. | ||
| * This Source Code Form is subject to the terms of the Mozilla Public | ||
| * License, v. 2.0. If a copy of the MPL was not distributed with this file, | ||
| * You can obtain one at http://mozilla.org/MPL/2.0/. */ | ||
|
|
||
| #ifndef BRAVE_VENDOR_BAT_NATIVE_ADS_SRC_BAT_ADS_INTERNAL_BASE_SEARCH_ENGINE_RESULTS_PAGE_URL_PATTERN_UTIL_H_ | ||
| #define BRAVE_VENDOR_BAT_NATIVE_ADS_SRC_BAT_ADS_INTERNAL_BASE_SEARCH_ENGINE_RESULTS_PAGE_URL_PATTERN_UTIL_H_ | ||
|
|
||
| #include <string> | ||
|
|
||
| namespace ads { | ||
|
|
||
| const std::string& GetAmazonResultsPageUrlPattern(); | ||
| const std::string& GetGoogleResultsPageUrlPattern(); | ||
| const std::string& GetMojeekResultsPageUrlPattern(); | ||
| const std::string& GetWikipediaResultsPageUrlPattern(); | ||
| const std::string& GetYahooResultsPageUrlPattern(); | ||
|
|
||
| } // namespace ads | ||
|
|
||
| #endif // BRAVE_VENDOR_BAT_NATIVE_ADS_SRC_BAT_ADS_INTERNAL_BASE_SEARCH_ENGINE_RESULTS_PAGE_URL_PATTERN_UTIL_H_ |
99 changes: 99 additions & 0 deletions
99
...ive-ads/src/bat/ads/internal/base/search_engine_results_page_url_pattern_util_unittest.cc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| /* Copyright (c) 2022 The Brave Authors. All rights reserved. | ||
| * This Source Code Form is subject to the terms of the Mozilla Public | ||
| * License, v. 2.0. If a copy of the MPL was not distributed with this file, | ||
| * You can obtain one at http://mozilla.org/MPL/2.0/. */ | ||
|
|
||
| #include "bat/ads/internal/base/search_engine_results_page_url_pattern_util.h" | ||
|
|
||
| #include "testing/gtest/include/gtest/gtest.h" | ||
|
|
||
| // npm run test -- brave_unit_tests --filter=BatAds* | ||
|
|
||
| namespace ads { | ||
|
|
||
| TEST(BatAdsSearchEngineResultsPageUrlPatternUtilTest, | ||
| GetAmazonResultsPageUrlPattern) { | ||
| // Arrange | ||
|
|
||
| // Act | ||
| const std::string& url_pattern = GetAmazonResultsPageUrlPattern(); | ||
|
|
||
| // Assert | ||
| EXPECT_EQ( | ||
| "https://" | ||
| "www.amazon.(ae|ca|cn|co.jp|co.uk|com|com.au|com.br|com.mx|de|eg|es|fr|" | ||
| "in|it|nl|pl|sa|se|sp|tr)/s", | ||
| url_pattern); | ||
| } | ||
|
|
||
| TEST(BatAdsSearchEngineResultsPageUrlPatternUtilTest, | ||
| GetGoogleResultsPageUrlPattern) { | ||
| // Arrange | ||
|
|
||
| // Act | ||
| const std::string& url_pattern = GetGoogleResultsPageUrlPattern(); | ||
|
|
||
| // Assert | ||
| EXPECT_EQ( | ||
| "https://" | ||
| "www.google.(ac|ad|ae|al|am|as|at|az|ba|be|bf|bg|bi|bj|bs|bt|ca|cat|cd|" | ||
| "cf|cg|ch|ci|ci|cl|cm|cn|co.bw|co.ck|co.cr|co.id|co.il|co.im|co.in|co.je|" | ||
| "co.jp|co.ke|co.kr|co.ls|co.ma|co.mz|co.nz|co.th|co.tz|co.ug|co.uk|co.uz|" | ||
| "co.ve|co.vi|co.za|co.zm|co.zw|com.af|com.ag|com.ai|com.ar|com.au|com.bd|" | ||
| "com.bh|com.bn|com.bo|com.br|com.by|com.bz|com.co|com.cu|com.cy|com.do|" | ||
| "com.ec|com.eg|com.et|com.fj|com.gh|com.gi|com.gt|com.hk|com.jm|com.kg|" | ||
| "com.kh|com.kw|com.lb|com.ly|com.mt|com.mx|com.my|com.na|com.nf|com.ng|" | ||
| "com.ni|com.np|com.om|com.pa|com.pe|com.pg|com.ph|com.pk|com.pr|com.py|" | ||
| "com.qa|com.sa|com.sb|com.sg|com.sl|com.sv|com.tj|com.tr|com.tw|com.ua|" | ||
| "com.uy|com.vc|com.vn|com|cv|cz|de|dj|dk|dm|dz|ee|es|fi|fm|fr|ga|ge|gg|" | ||
| "gl|gm|gp|gr|gy|hn|hr|ht|hu|ie|iq|is|it.ao|it|jo|ki|kz|la|li|lk|lt|lu|lv|" | ||
| "md|me|mg|mk|ml|mn|ms|mu|mv|mw|ne|nl|no|nr|nu|pl|pn|ps|pt|ro|rs|ru|rw|sc|" | ||
| "se|sh|si|sk|sm|sn|so|sr|st|td|tg|tk|tl|tm|tn|to|tt|vg|vu|ws)/search", | ||
| url_pattern); | ||
| } | ||
|
|
||
| TEST(BatAdsSearchEngineResultsPageUrlPatternUtilTest, | ||
| GetMojeekResultsPageUrlPattern) { | ||
| // Arrange | ||
|
|
||
| // Act | ||
| const std::string& url_pattern = GetMojeekResultsPageUrlPattern(); | ||
|
|
||
| // Assert | ||
| EXPECT_EQ("https://www.mojeek.(co.uk|com)/search", url_pattern); | ||
| } | ||
|
|
||
| TEST(BatAdsSearchEngineResultsPageUrlPatternUtilTest, | ||
| GetWikipediaResultsPageUrlPattern) { | ||
| // Arrange | ||
|
|
||
| // Act | ||
| const std::string& url_pattern = GetWikipediaResultsPageUrlPattern(); | ||
|
|
||
| // Assert | ||
| EXPECT_EQ( | ||
| "https://" | ||
| "(af|ar|arz|ast|az|azb|be|bg|bn|ca|ce|ceb|cs|cy|da|de|el|en|eo|es|et|eu|" | ||
| "fa|fi|fr|gl|he|hi|hr|hu|hy|id|it|ja|ka|kk|ko|la|lt|lv|min|mk|ms|my|nan|" | ||
| "nl|nn|no|pl|pt|ro|ru|sh|simple|sk|sl|sr|sv|ta|tg|th|tr|tt|uk|ur|uz|vi|" | ||
| "vo|war|zh|zh-yue).wikipedia.org/wiki/(.*)", | ||
| url_pattern); | ||
| } | ||
|
|
||
| TEST(BatAdsSearchEngineResultsPageUrlPatternUtilTest, | ||
| GetYahooResultsPageUrlPattern) { | ||
| // Arrange | ||
|
|
||
| // Act | ||
| const std::string& url_pattern = GetYahooResultsPageUrlPattern(); | ||
|
|
||
| // Assert | ||
| EXPECT_EQ( | ||
| "https://" | ||
| "((au|be|br|ca|de|en-maktoob|es|espanol|fr|fr-be|gr|hk|id|ie|in|it|" | ||
| "malaysia|nz|ph|qc|ro|se|sg|tw|uk|vn|www|za).search.yahoo.com/" | ||
| "|search.yahoo.com/)search(.*)", | ||
| url_pattern); | ||
| } | ||
|
|
||
| } // namespace ads |
Oops, something went wrong.