-
-
Notifications
You must be signed in to change notification settings - Fork 555
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #8 from DedSecInside/dev
Merge
- Loading branch information
Showing
21 changed files
with
553 additions
and
330 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,6 +17,8 @@ tests/.ropeproject/ | |
| *.pyc | ||
| .pytestc* | ||
| .pytest_cache | ||
| __pycache* | ||
| __pycache__/ | ||
|
|
||
| # Misc | ||
| torBot | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| ## FAQ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
|
|
||
| """ | ||
| Module containing class with colors | ||
| """ | ||
|
|
||
| class Colors: | ||
| """ | ||
| Class that contains colors used for TorBot in terminal and a method | ||
| that adds colr to a string | ||
| Attributes: | ||
| _colors (dict): A map containing all of the color codes needed | ||
| """ | ||
| def __init__(self): | ||
| self._colors = { | ||
| 'white': "\033[1;37m", | ||
| 'yellow': "\033[1;33m", | ||
| 'green': "\033[1;32m", | ||
| 'blue': "\033[1;34m", | ||
| 'cyan': "\033[1;36m", | ||
| 'red': "\033[1;31m", | ||
| 'magenta': "\033[1;35m", | ||
| 'black': "\033[1;30m", | ||
| 'darkwhite': "\033[0;37m", | ||
| 'darkyellow': "\033[0;33m", | ||
| 'darkgreen': "\033[0;32m", | ||
| 'darkblue': "\033[0;34m", | ||
| 'darkcyan': "\033[0;36m", | ||
| 'darkred': "\033[0;31m", | ||
| 'darkmagenta':"\033[0;35m", | ||
| 'darkblack': "\033[0;30m", | ||
| 'end': "\033[0;0m" | ||
| } | ||
|
|
||
| def add(self, string, color): | ||
| """ | ||
| Method that adds color to a given string | ||
| Args: | ||
| string (str): string to add color to | ||
| color (str): string of color to add | ||
| """ | ||
| return self.get(color) + string + self.get('end') | ||
|
|
||
| def get(self, color): | ||
| """ | ||
| Method that returns the color code of the given color string | ||
| Args: | ||
| color (str): color code to be returned | ||
| """ | ||
| return self._colors[color] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,129 @@ | ||
| from modules.net_utils import get_urls_from_page, get_url_status | ||
| from modules import pagereader | ||
|
|
||
| """ | ||
| Module used to interact with a pages urls | ||
| """ | ||
| import re | ||
|
|
||
| from bs4 import BeautifulSoup | ||
| from modules.bcolors import Bcolors | ||
|
|
||
| import modules.utils | ||
| import modules.pagereader | ||
|
|
||
| def add_green(link): | ||
| colors = Bcolors() | ||
| return '\t' + colors.OKGREEN + link + colors.ENDC | ||
| from modules.colors import Colors | ||
|
|
||
| COLOR = Colors() | ||
|
|
||
| def add_red(link): | ||
| colors = Bcolors() | ||
| return '\t' + colors.On_Red + link + colors.ENDC | ||
| def is_url(url): | ||
| """ | ||
| Returns an integer representing validity of url syntax | ||
| Args: | ||
| url (str): url to be verified | ||
| Returns | ||
| (int): integer representing if url is a valid format | ||
| """ | ||
| pattern = r"^https?:\/\/(www\.)?([a-z,A-Z,0-9]*)\.([a-z, A-Z]+)(.*)" | ||
| regex = re.compile(pattern) | ||
| if regex.match(url): | ||
| return 1 | ||
| return 0 | ||
|
|
||
| def get_links(soup, ext=False, live=False): | ||
|
|
||
| def is_onion_url(url): | ||
| """ | ||
| Searches through all <a ref> (hyperlinks) tags and stores them in a | ||
| list then validates if the url is formatted correctly. | ||
| Returns an integer representing validity of an onion url syntax | ||
| Args: | ||
| soup: BeautifulSoup instance currently being used. | ||
| Args: | ||
| url (str): url to be verified | ||
| Returns | ||
| (int): integer representing if url is a valid format | ||
| """ | ||
| pattern = r"^https?:\/\/(www\.)?([a-z,A-Z,0-9]*)\.onion/(.*)" | ||
| regex = re.compile(pattern) | ||
| if regex.match(url): | ||
| return 1 | ||
| return 0 | ||
|
|
||
| def get_urls_from_page(page_soup, email=False, extension=False): | ||
| """ | ||
| Searches for urls on page using the anchor tag and href attribute, | ||
| also searchs for emails using 'mailto' if specified. | ||
| Args: | ||
| page (bs4.BeauitulSoup): html soup to search | ||
| email (bool): flag whether to collect emails as well | ||
| extension (bool): flag whether to use additional extensions | ||
| Returns: | ||
| urls (list): urls found on page | ||
| """ | ||
| if not isinstance(page_soup, BeautifulSoup): | ||
| raise Exception("First arg must be bs4.BeautifulSoup object") | ||
|
|
||
| urls = [] | ||
| anchors_on_page = page_soup.find_all('a') | ||
| for anchor_tag in anchors_on_page: | ||
| url = anchor_tag.get('href') | ||
| if extension: | ||
| if url and is_url(url) == 1: | ||
| urls.append(url) | ||
| elif email: | ||
| if url and 'mailto' in url: | ||
| email_addr = url.split(':') | ||
| if len(email_addr) > 1: | ||
| urls.append(email_addr[1]) | ||
| else: | ||
| if url and is_onion_url(url) == 1: | ||
| urls.append(url) | ||
|
|
||
| return urls | ||
|
|
||
|
|
||
| def search_page(html, ext, stop_depth=None): | ||
| """ | ||
| Takes in a pages HTML and searches the links on the page using | ||
| BFS. | ||
| Args: | ||
| html (str): HTML with links to search | ||
| add_exts (str): additional extension | ||
| stop_depth (int): The depth at which to stop | ||
| Returns: | ||
| websites: List of websites that were found | ||
| links_found (list): links found on page and associated pages | ||
| """ | ||
|
|
||
| soup = BeautifulSoup(html, 'html.parser') | ||
| links = get_urls_from_page(soup, extension=ext) | ||
| if stop_depth: | ||
| links_found = modules.utils.bfs_urls(links, ext, stop_depth=stop_depth) | ||
| else: | ||
| links_found = modules.utils.bfs_urls(links, ext) | ||
|
|
||
| return links_found | ||
|
|
||
|
|
||
| def get_links(soup, ext=False, live=False): | ||
| """ | ||
| Returns list of links listed on the webpage of the soup passed. If live | ||
| is set to true then it will also print the status of each of the links | ||
| and setting ext to an actual extension such as '.com' will allow those | ||
| extensions to be recognized as valid urls and not just '.tor'. | ||
| Args: | ||
| soup (bs4.BeautifulSoup): webpage to be searched for links. | ||
| Returns: | ||
| websites (list(str)): List of websites that were found | ||
| """ | ||
| b_colors = Bcolors() | ||
| if isinstance(soup, BeautifulSoup): | ||
| websites = get_urls_from_page(soup, extension=ext) | ||
| # Pretty print output as below | ||
| print(''.join((b_colors.OKGREEN, | ||
| 'Websites Found - ', b_colors.ENDC, str(len(websites))))) | ||
| success_string = 'Websites Found - ' + str(len(websites)) | ||
| print(COLOR.add(success_string, 'green')) | ||
| print('------------------------------------') | ||
|
|
||
| if live: | ||
| for link in websites: | ||
| if get_url_status(link) != 0: | ||
| coloredlink = add_green(link) | ||
| page = pagereader.read_first_page(link)[0] | ||
| if page is not None and page.title is not None: | ||
| print_row(coloredlink, page.title.string) | ||
| else: | ||
| coloredlink = add_red(link) | ||
| print_row(coloredlink, "Not found") | ||
|
|
||
| modules.utils.queue_tasks(websites, modules.pagereader.display_url) | ||
| return websites | ||
|
|
||
| else: | ||
| raise(Exception('Method parameter is not of instance BeautifulSoup')) | ||
|
|
||
|
|
||
| def print_row(url, description): | ||
| print("%-80s %-30s" % (url, description)) | ||
| raise Exception('Method parameter is not of instance BeautifulSoup') |
Oops, something went wrong.