diff --git a/projectIndexer.py b/projectIndexer.py index 502a147..e383978 100644 --- a/projectIndexer.py +++ b/projectIndexer.py @@ -18,15 +18,16 @@ def cli(): @click.option('--github-token', default=None, help='Github token (default is in .env file)') @click.option('--output-repos', '-o', default='data/repos.json', help='build file (default is repos.json)') @click.option('--output-readme', '-r', default='data/readme.json', help='build file (default is readme.json)') +@click.option('--output-contributors', '-r', default='data/contributors.json', help='build file (default is contributors.json)') @click.option('--verbose', default=False, is_flag=True, help='Verbose build') -def fetch_github(github_token, output_repos, output_readme, verbose): +def fetch_github(github_token, output_repos, output_readme, output_contributors, verbose): fetch_data = FetchData(github_token) start = time() repos = fetch_data.fetch_repos() if repos is None: logger.error("No repos fetched") return - repos_count = fetch_data.save_to_file(repos, output_repos, output_readme, verbose) + repos_count = fetch_data.save_to_file(repos, output_repos, output_readme, output_contributors, verbose) logger.info(f"Fetched {repos_count} repos in {time() - start:.2f} seconds") @click.command(help='List repos from file - for debugging') @@ -46,6 +47,7 @@ def list(github_token, input_file): @click.option('--fetch-directly', default=False, is_flag=True, help='Fetch repos directly from github') @click.option('--input-repos', default='data/repos.json', help='Input file (default is repos.json)') @click.option('--input-readme', default='data/readme.json', help='Input file (default is readme.json)') +@click.option('--input-contributors', default='data/contributors.json', help='Input file (default is contributors.json)') @click.option('--build-dir', '-o', default='build', help='build directory (default is build)') @click.option('--template-dir', '-t', default='templates', help="Template directory (default is 'templates')") @click.option('--static-dir', default='static', help="Static directory (default is 'static')") @@ -53,7 +55,7 @@ def list(github_token, input_file): @click.option('--hide-private', default=False, is_flag=True, help="Hide private repos") @click.option('--verbose', default=False, is_flag=True, help='Verbose build') @click.option('--compile-tailwind', default=False, is_flag=True, help='Compile tailwind (requires npx + tailwindcss)') -def generate(github_token: str, fetch_directly: bool, input_repos: str, input_readme:str, build_dir: str, template_dir: str, static_dir: str, project_dir: str, hide_private: bool, verbose: bool, compile_tailwind: bool): +def generate(github_token: str, fetch_directly: bool, input_repos: str, input_readme:str, input_contributors:str, build_dir: str, template_dir: str, static_dir: str, project_dir: str, hide_private: bool, verbose: bool, compile_tailwind: bool): print(f"Generating web to {build_dir} directory") start = time() fetch_data = FetchData(github_token) @@ -66,8 +68,9 @@ def generate(github_token: str, fetch_directly: bool, input_repos: str, input_re else: repos = fetch_data.load_repo(input_repos) readme = fetch_data.load_readme(input_readme) + contributors = fetch_data.load_contributors(input_contributors) - if not repos or not readme: + if not repos or not readme or not contributors: raise Exception("No repos loaded") # sort repos by time of last push (newest to oldest) @@ -80,7 +83,7 @@ def generate(github_token: str, fetch_directly: bool, input_repos: str, input_re for repo, t in sorted_repos: repos.append(repo) - generate_web = GenerateWeb(repos, readme, build_dir, path.abspath(template_dir), static_dir, project_dir, hide_private, verbose, compile_tailwind) + generate_web = GenerateWeb(repos, readme, contributors, build_dir, path.abspath(template_dir), static_dir, project_dir, hide_private, verbose, compile_tailwind) generate_web.generate() print(f"Generated web to {build_dir} directory in {time() - start:.2f} seconds") diff --git a/src/fetch_data.py b/src/fetch_data.py index d37dc2b..a0bdfed 100644 --- a/src/fetch_data.py +++ b/src/fetch_data.py @@ -26,9 +26,14 @@ def fetch_readme(self, repo: Repository.Repository) -> str: return repo.get_readme().decoded_content.decode("utf-8") except UnknownObjectException: return "" + + def fetch_contributors(self, repo: Repository.Repository, contributors_limit:int = 15) -> PaginatedList: + try: + return sorted(repo.get_contributors(), key=lambda x: x.contributions, reverse=True)[:contributors_limit] + except UnknownObjectException: + return [] - - def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data/repos.json', file_readme:str= 'data/readme.json', verbose=False) -> int: + def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data/repos.json', file_readme:str= 'data/readme.json', file_contributors:str= 'data/contributors.json', verbose=False) -> int: """ Save repos to file :param file_repos: @@ -42,6 +47,7 @@ def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data data_repo = [] data_readme = {} + data_contrib = {} index = 0 for repo in repos: if verbose: @@ -50,6 +56,8 @@ def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data data_readme[repo.full_name] = self.fetch_readme(repo) + contributors = self.fetch_contributors(repo) + data_contrib[repo.full_name] = [[contributor.html_url, contributor.avatar_url, contributor.name, contributor.login, contributor.contributions,] for contributor in contributors] index += 1 with open(file_repos, 'w') as json_file: json.dump(data_repo, json_file) @@ -57,6 +65,9 @@ def save_to_file(self, repos: list[Repository.Repository], file_repos:str= 'data with open(file_readme, 'w') as json_file: json.dump(data_readme, json_file) + with open(file_contributors, 'w') as json_file: + json.dump(data_contrib, json_file) + if verbose: logger.info(f"Saved {index} repos to {file_repos}") @@ -81,6 +92,15 @@ def load_readme(self, file_readme: str="data/readme.json") -> dict[str, str]: except FileNotFoundError: logger.error(f"File {file_readme} not found") return {} + + def load_contributors(self, file_contrib: str="data/contributors.json") -> dict[str, list[str]]: + # return {repo, [name, login, url_to_img]} + try: + with open(file_contrib, 'r') as json_file: + return json.load(json_file) + except FileNotFoundError: + logger.error(f"File {file_contrib} not found") + return {} if __name__ == "__main__": fetch_data = FetchData(environ.get('MY_GITHUB_TOKEN')) diff --git a/src/generate_web.py b/src/generate_web.py index 68502c2..47a42e7 100644 --- a/src/generate_web.py +++ b/src/generate_web.py @@ -25,6 +25,7 @@ def __init__( self, repos: list[Repository.Repository], readme: dict[str, str], + contributors: dict[list], build_dir: str = 'build', template_dir: path = 'templates', static_dir: path = 'static', @@ -39,6 +40,7 @@ def __init__( self.repos = repos self.readme = readme + self.contributors = contributors self.static_dir = static_dir self.template_dir = template_dir self.project_dir = project_dir @@ -114,14 +116,17 @@ def generate_repos_list(self): self.render_page('repos.html', self.paths.get("Repos").get("path"), repos=self.repos) def generate_repos_detail(self): - for repo in self.repos: + repo_count = len(self.repos) + for i, repo in enumerate(self.repos): + print(f"Generating {repo.name} {i}/{repo_count}") readme_md = self.readme.get(repo.full_name, "No readme found") readme_fixed_images = fix_readme_relative_images(readme_md, repo.full_name, repo.default_branch) readme_fixed_images = readme_fixed_images.replace(':\n- ', ':\n\n- ') # fix markdown lists readme_fixed_images = readme_fixed_images.replace('- ', '- ● ') # add the dot before each element of the list readme_html = markdown(readme_fixed_images, extensions=['fenced_code']) path_repo = self.paths.get("Repo").get("path").format(repo.name) - self.render_page('repoDetail.html', path_repo, repo=repo, readme=readme_html) + + self.render_page('repoDetail.html', path_repo, repo=repo, readme=readme_html, repo_contrib = self.contributors[repo.full_name]) def generate_demo(self): self.render_page('demo.html', self.paths.get("Demo").get("path")) diff --git a/src/web_helpers.py b/src/web_helpers.py index ffc473a..380b5d4 100644 --- a/src/web_helpers.py +++ b/src/web_helpers.py @@ -52,10 +52,14 @@ def replace_url(img_url, alt_text=""): new_url = base_url + img_url elif img_url.startswith("./"): new_url = f"{base_url}/{img_url[2:]}" - else: + elif img_url.startswith("https://github.com"): + new_url = base_url + img_url.split("/blob/master")[-1] + elif img_url.startswith("http"): # The URL is already absolute, or it's a type of relative URL - # that we aren't handling. new_url = img_url + else: + new_url = base_url + "/"+ img_url + return f"![{alt_text}]({new_url})" if alt_text else new_url # Replace Markdown image URLs diff --git a/templates/repoDetail.html b/templates/repoDetail.html index 80d09d7..92c3923 100644 --- a/templates/repoDetail.html +++ b/templates/repoDetail.html @@ -1,7 +1,7 @@ {% set title = repo.name %} {% extends "base.html" %} {% block content %} -
+
{# Repo info #}
@@ -52,16 +52,15 @@

{{ repo.name }}

{# Contributors #} -
- {% for contrib in repo.get_contributors() %} - - +
+ {% for contrib in repo_contrib %} + +
-

{{ contrib.name }}

-

{{ contrib.login }}

-

{{ contrib.contributions }} {% if - contrib.contributions +

{{ contrib[2] }}

+

{{ contrib[3]}}

+

{{ contrib[4] }} {% if + contrib[3] == 1 %}commit{% else %}commits{% endif %}