diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 229856aa4366..000000000000 --- a/.flake8 +++ /dev/null @@ -1,22 +0,0 @@ -[flake8] -ignore = - ;W503 line break before binary operator - W503, - ;E203 whitespace before ':' - E203, - -; exclude file -exclude = - .tox, - .git, - __pycache__, - build, - dist, - *.pyc, - *.egg-info, - .cache, - .eggs - -max-line-length = 120 - -per-file-ignores = __init__.py:F401 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index b310fcfefc15..436bdf887c69 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ blank_issues_enabled: true contact_links: - name: ❓ Simple question - Slack Chat - url: https://join.slack.com/t/colossalaiworkspace/shared_invite/zt-z7b26eeb-CBp7jouvu~r0~lcFzX832w + url: https://github.com/hpcaitech/public_assets/tree/main/colossalai/contact/slack about: This issue tracker is not for technical support. Please use our Slack chat, and ask the community for help. - name: ❓ Simple question - WeChat url: https://github.com/hpcaitech/ColossalAI/blob/main/docs/images/WeChat.png diff --git a/.github/workflows/build_on_pr.yml b/.github/workflows/build_on_pr.yml index 291d6adac2b2..e2114d43bcd0 100644 --- a/.github/workflows/build_on_pr.yml +++ b/.github/workflows/build_on_pr.yml @@ -141,7 +141,7 @@ jobs: runs-on: [self-hosted, gpu] container: image: hpcaitech/pytorch-cuda:1.12.0-11.3.0 - options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 + options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 -v /data/scratch/llama-tiny:/data/scratch/llama-tiny timeout-minutes: 60 defaults: run: @@ -214,6 +214,7 @@ jobs: NCCL_SHM_DISABLE: 1 LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 TESTMON_CORE_PKGS: /__w/ColossalAI/ColossalAI/requirements/requirements.txt,/__w/ColossalAI/ColossalAI/requirements/requirements-test.txt + LLAMA_PATH: /data/scratch/llama-tiny - name: Store Testmon Cache run: | diff --git a/.github/workflows/build_on_schedule.yml b/.github/workflows/build_on_schedule.yml index 03b47e6cb5b6..6c77377be34f 100644 --- a/.github/workflows/build_on_schedule.yml +++ b/.github/workflows/build_on_schedule.yml @@ -13,7 +13,7 @@ jobs: runs-on: [self-hosted, 8-gpu] container: image: hpcaitech/pytorch-cuda:1.12.0-11.3.0 - options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 + options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 -v /data/scratch/llama-tiny:/data/scratch/llama-tiny timeout-minutes: 40 steps: - name: Check GPU Availability # ensure all GPUs have enough memory @@ -64,6 +64,7 @@ jobs: env: DATA: /data/scratch/cifar-10 LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 + LLAMA_PATH: /data/scratch/llama-tiny - name: Notify Lark id: message-preparation diff --git a/.github/workflows/compatiblity_test_on_dispatch.yml b/.github/workflows/compatiblity_test_on_dispatch.yml index 2f03c8ced98d..5083212993cc 100644 --- a/.github/workflows/compatiblity_test_on_dispatch.yml +++ b/.github/workflows/compatiblity_test_on_dispatch.yml @@ -50,7 +50,7 @@ jobs: matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}} container: image: ${{ matrix.container }} - options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 + options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 -v /data/scratch/llama-tiny:/data/scratch/llama-tiny timeout-minutes: 120 steps: - name: Install dependencies @@ -92,3 +92,4 @@ jobs: DATA: /data/scratch/cifar-10 NCCL_SHM_DISABLE: 1 LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 + LLAMA_PATH: /data/scratch/llama-tiny diff --git a/.github/workflows/compatiblity_test_on_pr.yml b/.github/workflows/compatiblity_test_on_pr.yml index a621c7e3427d..cc17c66f9c3a 100644 --- a/.github/workflows/compatiblity_test_on_pr.yml +++ b/.github/workflows/compatiblity_test_on_pr.yml @@ -41,7 +41,7 @@ jobs: matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}} container: image: ${{ matrix.container }} - options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 + options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 -v /data/scratch/llama-tiny:/data/scratch/llama-tiny timeout-minutes: 120 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-run-test-${{ matrix.container }} @@ -87,3 +87,4 @@ jobs: DATA: /data/scratch/cifar-10 NCCL_SHM_DISABLE: 1 LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 + LLAMA_PATH: /data/scratch/llama-tiny diff --git a/.github/workflows/compatiblity_test_on_schedule.yml b/.github/workflows/compatiblity_test_on_schedule.yml index 9933224f5675..158fe751bf2e 100644 --- a/.github/workflows/compatiblity_test_on_schedule.yml +++ b/.github/workflows/compatiblity_test_on_schedule.yml @@ -38,7 +38,7 @@ jobs: matrix: ${{fromJson(needs.matrix_preparation.outputs.matrix)}} container: image: ${{ matrix.container }} - options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 + options: --gpus all --rm -v /data/scratch/cifar-10:/data/scratch/cifar-10 -v /data/scratch/llama-tiny:/data/scratch/llama-tiny timeout-minutes: 120 steps: - name: Install dependencies @@ -85,6 +85,7 @@ jobs: DATA: /data/scratch/cifar-10 NCCL_SHM_DISABLE: 1 LD_LIBRARY_PATH: /github/home/.tensornvme/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 + LLAMA_PATH: /data/scratch/llama-tiny - name: Notify Lark id: message-preparation diff --git a/.github/workflows/doc_test_on_pr.yml b/.github/workflows/doc_test_on_pr.yml index a3df2c50e6d3..f1e7a2d0cab0 100644 --- a/.github/workflows/doc_test_on_pr.yml +++ b/.github/workflows/doc_test_on_pr.yml @@ -89,7 +89,7 @@ jobs: - name: Install ColossalAI run: | source activate pytorch - pip install -v . + CUDA_EXT=1 pip install -v . - name: Test the Doc run: | diff --git a/.github/workflows/doc_test_on_schedule.yml b/.github/workflows/doc_test_on_schedule.yml index 6b4f5d1f908c..027fbfd0aaeb 100644 --- a/.github/workflows/doc_test_on_schedule.yml +++ b/.github/workflows/doc_test_on_schedule.yml @@ -32,7 +32,7 @@ jobs: - name: Install ColossalAI run: | - pip install -v . + CUDA_EXT=1 pip install -v . - name: Install Doc Test Requirements run: | diff --git a/.github/workflows/example_check_on_dispatch.yml b/.github/workflows/example_check_on_dispatch.yml index 620d4771af55..9d3bd9a48235 100644 --- a/.github/workflows/example_check_on_dispatch.yml +++ b/.github/workflows/example_check_on_dispatch.yml @@ -53,7 +53,7 @@ jobs: uses: actions/checkout@v3 - name: Install Colossal-AI run: | - pip install -v . + CUDA_EXT=1 pip install -v . - name: Test the example run: | dir=${{ matrix.directory }} diff --git a/.github/workflows/example_check_on_pr.yml b/.github/workflows/example_check_on_pr.yml index ec23b9d1c59f..5934704f4102 100644 --- a/.github/workflows/example_check_on_pr.yml +++ b/.github/workflows/example_check_on_pr.yml @@ -88,7 +88,7 @@ jobs: - name: Install Colossal-AI run: | - pip install -v . + CUDA_EXT=1 pip install -v . - name: Test the example run: | diff --git a/.github/workflows/example_check_on_schedule.yml b/.github/workflows/example_check_on_schedule.yml index bd52ca4321a2..5ed128c3ebc5 100644 --- a/.github/workflows/example_check_on_schedule.yml +++ b/.github/workflows/example_check_on_schedule.yml @@ -42,7 +42,7 @@ jobs: - name: Install Colossal-AI run: | - pip install -v . + CUDA_EXT=1 pip install -v . - name: Traverse all files run: | diff --git a/.github/workflows/run_chatgpt_examples.yml b/.github/workflows/run_chatgpt_examples.yml index a336526897e2..f9e9f400962e 100644 --- a/.github/workflows/run_chatgpt_examples.yml +++ b/.github/workflows/run_chatgpt_examples.yml @@ -49,5 +49,5 @@ jobs: NCCL_SHM_DISABLE: 1 MAX_JOBS: 8 SFT_DATASET: /data/scratch/github_actions/chat/data.json - PROMPT_PATH: /data/scratch/github_actions/chat/prompts_en.jsonl + PROMPT_DATASET: /data/scratch/github_actions/chat/prompts_en.jsonl PRETRAIN_DATASET: /data/scratch/github_actions/chat/alpaca_data.json diff --git a/.github/workflows/scripts/check_doc_i18n.py b/.github/workflows/scripts/check_doc_i18n.py index 1aa7283e9e52..1e7f0c33a785 100644 --- a/.github/workflows/scripts/check_doc_i18n.py +++ b/.github/workflows/scripts/check_doc_i18n.py @@ -22,13 +22,13 @@ def compare_dirs(dir1, dir2): # If the corresponding item doesn't exist in the second directory, the directories are different if not os.path.exists(item_path2): - print(f'Found mismatch: {item_path1}, {item_path2}') + print(f"Found mismatch: {item_path1}, {item_path2}") return False # If the corresponding item is a directory, we compare the two directories recursively if os.path.isdir(item_path1) and os.path.isdir(item_path2): if not compare_dirs(item_path1, item_path2): - print(f'Found mismatch: {item_path1}, {item_path2}') + print(f"Found mismatch: {item_path1}, {item_path2}") return False # both are files @@ -37,16 +37,16 @@ def compare_dirs(dir1, dir2): # If the corresponding item is not a file or a directory, the directories are different else: - print(f'Found mismatch: {item_path1}, {item_path2}') + print(f"Found mismatch: {item_path1}, {item_path2}") return False # If all items are the same, the directories are the same return True -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('-d', '--directory', help="The directory where the multi-language source files are kept.") + parser.add_argument("-d", "--directory", help="The directory where the multi-language source files are kept.") args = parser.parse_args() i18n_folders = os.listdir(args.directory) @@ -56,7 +56,7 @@ def compare_dirs(dir1, dir2): for i in range(1, len(i18n_folders)): dir1 = i18n_folders[0] dir2 = i18n_folders[i] - print(f'comparing {dir1} vs {dir2}') + print(f"comparing {dir1} vs {dir2}") match = compare_dirs(i18n_folders[0], i18n_folders[i]) if not match: diff --git a/.github/workflows/scripts/example_checks/check_dispatch_inputs.py b/.github/workflows/scripts/example_checks/check_dispatch_inputs.py index 5bec96187e0c..91778f692cc6 100644 --- a/.github/workflows/scripts/example_checks/check_dispatch_inputs.py +++ b/.github/workflows/scripts/example_checks/check_dispatch_inputs.py @@ -4,7 +4,7 @@ def check_inputs(input_list): for path in input_list: - real_path = os.path.join('examples', path) + real_path = os.path.join("examples", path) if not os.path.exists(real_path): return False return True @@ -12,16 +12,16 @@ def check_inputs(input_list): def main(): parser = argparse.ArgumentParser() - parser.add_argument('-f', '--fileNameList', type=str, help="List of file names") + parser.add_argument("-f", "--fileNameList", type=str, help="List of file names") args = parser.parse_args() name_list = args.fileNameList.split(",") is_correct = check_inputs(name_list) if is_correct: - print('success') + print("success") else: - print('failure') + print("failure") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/.github/workflows/scripts/example_checks/check_example_weekly.py b/.github/workflows/scripts/example_checks/check_example_weekly.py index 83eff644e315..95a3d24c9a78 100644 --- a/.github/workflows/scripts/example_checks/check_example_weekly.py +++ b/.github/workflows/scripts/example_checks/check_example_weekly.py @@ -17,21 +17,21 @@ def show_files(path, all_files): def join(input_list, sep=None): - return (sep or ' ').join(input_list) + return (sep or " ").join(input_list) def main(): - contents = show_files('examples/', []) + contents = show_files("examples/", []) all_loc = [] for file_loc in contents: - split_loc = file_loc.split('/') + split_loc = file_loc.split("/") # must have two sub-folder levels after examples folder, such as examples/images/vit is acceptable, examples/images/README.md is not, examples/requirements.txt is not. if len(split_loc) >= 4: - re_loc = '/'.join(split_loc[1:3]) + re_loc = "/".join(split_loc[1:3]) if re_loc not in all_loc: all_loc.append(re_loc) print(all_loc) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/.github/workflows/scripts/example_checks/detect_changed_example.py b/.github/workflows/scripts/example_checks/detect_changed_example.py index c69d95a552e9..95f671dfb32b 100644 --- a/.github/workflows/scripts/example_checks/detect_changed_example.py +++ b/.github/workflows/scripts/example_checks/detect_changed_example.py @@ -3,7 +3,7 @@ def main(): parser = argparse.ArgumentParser() - parser.add_argument('-f', '--fileNameList', type=str, help="The list of changed files") + parser.add_argument("-f", "--fileNameList", type=str, help="The list of changed files") args = parser.parse_args() name_list = args.fileNameList.split(":") folder_need_check = set() @@ -15,10 +15,10 @@ def main(): # - application # - file if loc.split("/")[0] == "examples" and len(loc.split("/")) >= 4: - folder_need_check.add('/'.join(loc.split("/")[1:3])) + folder_need_check.add("/".join(loc.split("/")[1:3])) # Output the result using print. Then the shell can get the values. print(list(folder_need_check)) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/.github/workflows/scripts/generate_leaderboard_and_send_to_lark.py b/.github/workflows/scripts/generate_leaderboard_and_send_to_lark.py index 2884e38dd3dd..412b14c7b283 100644 --- a/.github/workflows/scripts/generate_leaderboard_and_send_to_lark.py +++ b/.github/workflows/scripts/generate_leaderboard_and_send_to_lark.py @@ -74,16 +74,16 @@ def get_organization_repositories(github_token, organization_name) -> List[str]: # prepare header headers = { - 'Authorization': f'Bearer {github_token}', - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28' + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", } res = requests.get(url, headers=headers).json() repo_list = [] for item in res: - repo_list.append(item['name']) + repo_list.append(item["name"]) return repo_list @@ -97,9 +97,9 @@ def get_issue_pull_request_comments(github_token: str, org_name: str, repo_name: """ # prepare header headers = { - 'Authorization': f'Bearer {github_token}', - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28' + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", } user_engagement_count = {} @@ -107,28 +107,28 @@ def get_issue_pull_request_comments(github_token: str, org_name: str, repo_name: # do pagination to the API page = 1 while True: - comment_api = f'https://api.github.com/repos/{org_name}/{repo_name}/issues/comments?since={since}&page={page}' + comment_api = f"https://api.github.com/repos/{org_name}/{repo_name}/issues/comments?since={since}&page={page}" comment_response = requests.get(comment_api, headers=headers).json() if len(comment_response) == 0: break else: for item in comment_response: - comment_author_relationship = item['author_association'] - if comment_author_relationship != 'MEMBER': + comment_author_relationship = item["author_association"] + if comment_author_relationship != "MEMBER": # if the comment is not made by our member # we don't count this comment towards user engagement continue - issue_id = item['issue_url'].split('/')[-1] - issue_api = f'https://api.github.com/repos/{org_name}/{repo_name}/issues/{issue_id}' + issue_id = item["issue_url"].split("/")[-1] + issue_api = f"https://api.github.com/repos/{org_name}/{repo_name}/issues/{issue_id}" issue_response = requests.get(issue_api, headers=headers).json() - issue_author_relationship = issue_response['author_association'] + issue_author_relationship = issue_response["author_association"] - if issue_author_relationship != 'MEMBER': + if issue_author_relationship != "MEMBER": # this means that the issue/PR is not created by our own people # any comments in this issue/PR by our member will be counted towards the leaderboard - member_name = item['user']['login'] + member_name = item["user"]["login"] if member_name in user_engagement_count: user_engagement_count[member_name] += 1 @@ -153,7 +153,7 @@ def _generate_discussion_query(num, cursor: str = None): if cursor is None: offset_str = "" else: - offset_str = f", after: \"{cursor}\"" + offset_str = f', after: "{cursor}"' query = f""" {{ repository(owner: "{org_name}", name: "{repo_name}"){{ @@ -182,7 +182,7 @@ def _generate_comment_reply_count_for_discussion(discussion_number, num, cursor: if cursor is None: offset_str = "" else: - offset_str = f", before: \"{cursor}\"" + offset_str = f', before: "{cursor}"' query = f""" {{ repository(owner: "{org_name}", name: "{repo_name}"){{ @@ -220,8 +220,8 @@ def _generate_comment_reply_count_for_discussion(discussion_number, num, cursor: # a utility function to make call to Github GraphQL API def _call_graphql_api(query): headers = {"Authorization": f"Bearer {github_token}"} - json_data = {'query': query} - response = requests.post('https://api.github.com/graphql', json=json_data, headers=headers) + json_data = {"query": query} + response = requests.post("https://api.github.com/graphql", json=json_data, headers=headers) data = response.json() return data @@ -234,21 +234,21 @@ def _call_graphql_api(query): data = _call_graphql_api(query) found_discussion_out_of_time_range = False - edges = data['data']['repository']['discussions']['edges'] + edges = data["data"]["repository"]["discussions"]["edges"] if len(edges) == 0: break else: # keep the discussion whose author is not a member for edge in edges: # print the discussion title - discussion = edge['node'] - discussion_updated_at = str2datetime(discussion['updatedAt']) + discussion = edge["node"] + discussion_updated_at = str2datetime(discussion["updatedAt"]) # check if the updatedAt is within the last 7 days # if yes, add it to discussion_numbers if discussion_updated_at > since: - if discussion['authorAssociation'] != 'MEMBER': - discussion_numbers.append(discussion['number']) + if discussion["authorAssociation"] != "MEMBER": + discussion_numbers.append(discussion["number"]) else: found_discussion_out_of_time_range = True @@ -256,7 +256,7 @@ def _call_graphql_api(query): break else: # update cursor - cursor = edges[-1]['cursor'] + cursor = edges[-1]["cursor"] # get the discussion comments and replies made by our member user_engagement_count = {} @@ -269,42 +269,42 @@ def _call_graphql_api(query): data = _call_graphql_api(query) # get the comments - edges = data['data']['repository']['discussion']['comments']['edges'] + edges = data["data"]["repository"]["discussion"]["comments"]["edges"] # update the cursor if len(edges) == 0: break else: # update cursor for pagination - cursor = edges[-1]['cursor'] + cursor = edges[-1]["cursor"] for edge in edges: - comment = edge['node'] - if comment['authorAssociation'] == 'MEMBER': + comment = edge["node"] + if comment["authorAssociation"] == "MEMBER": # check if the updatedAt is within the last 7 days # if yes, add it to user_engagement_count - comment_updated_at = datetime.strptime(comment['updatedAt'], "%Y-%m-%dT%H:%M:%SZ") + comment_updated_at = datetime.strptime(comment["updatedAt"], "%Y-%m-%dT%H:%M:%SZ") if comment_updated_at > since: - member_name = comment['author']['login'] + member_name = comment["author"]["login"] if member_name in user_engagement_count: user_engagement_count[member_name] += 1 else: user_engagement_count[member_name] = 1 # get the replies - reply_edges = comment['replies']['edges'] + reply_edges = comment["replies"]["edges"] if len(reply_edges) == 0: continue else: for reply_edge in reply_edges: - reply = reply_edge['node'] - if reply['authorAssociation'] == 'MEMBER': + reply = reply_edge["node"] + if reply["authorAssociation"] == "MEMBER": # check if the updatedAt is within the last 7 days # if yes, add it to discussion_numbers - reply_updated_at = datetime.strptime(reply['updatedAt'], "%Y-%m-%dT%H:%M:%SZ") + reply_updated_at = datetime.strptime(reply["updatedAt"], "%Y-%m-%dT%H:%M:%SZ") if reply_updated_at > since: - member_name = reply['author']['login'] + member_name = reply["author"]["login"] if member_name in user_engagement_count: user_engagement_count[member_name] += 1 else: @@ -312,7 +312,9 @@ def _call_graphql_api(query): return user_engagement_count -def generate_user_engagement_leaderboard_image(github_token: str, org_name: str, repo_list: List[str], output_path: str) -> bool: +def generate_user_engagement_leaderboard_image( + github_token: str, org_name: str, repo_list: List[str], output_path: str +) -> bool: """ Generate the user engagement leaderboard image for stats within the last 7 days @@ -335,16 +337,19 @@ def _update_count(counter): else: total_engagement_count[name] = count - for repo_name in repo_list: print(f"Fetching user engagement count for {repo_name}/{repo_name}") - issue_pr_engagement_count = get_issue_pull_request_comments(github_token=github_token, org_name=org_name, repo_name=repo_name, since=start_datetime_str) - discussion_engagement_count = get_discussion_comments(github_token=github_token, org_name=org_name, repo_name=repo_name, since=start_datetime) + issue_pr_engagement_count = get_issue_pull_request_comments( + github_token=github_token, org_name=org_name, repo_name=repo_name, since=start_datetime_str + ) + discussion_engagement_count = get_discussion_comments( + github_token=github_token, org_name=org_name, repo_name=repo_name, since=start_datetime + ) # update the total engagement count _update_count(issue_pr_engagement_count) _update_count(discussion_engagement_count) - + # prepare the data for plotting x = [] y = [] @@ -363,7 +368,7 @@ def _update_count(counter): # plot the leaderboard xlabel = f"Number of Comments made (since {start_datetime_str})" ylabel = "Member" - title = 'Active User Engagement Leaderboard' + title = "Active User Engagement Leaderboard" plot_bar_chart(x, y, xlabel=xlabel, ylabel=ylabel, title=title, output_path=output_path) return True else: @@ -380,16 +385,16 @@ def generate_contributor_leaderboard_image(github_token, org_name, repo_list, ou """ # request to the Github API to get the users who have contributed in the last 7 days headers = { - 'Authorization': f'Bearer {github_token}', - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28' + "Authorization": f"Bearer {github_token}", + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", } counter = Counter() start_datetime = get_utc_time_one_week_ago() def _get_url(org_name, repo_name, page): - return f'https://api.github.com/repos/{org_name}/{repo_name}/pulls?per_page=50&page={page}&state=closed' + return f"https://api.github.com/repos/{org_name}/{repo_name}/pulls?per_page=50&page={page}&state=closed" def _iterate_by_page(org_name, repo_name): page = 1 @@ -415,8 +420,8 @@ def _iterate_by_page(org_name, repo_name): # count the pull request and author from response for pr_data in response: - merged_at = pr_data['merged_at'] - author = pr_data['user']['login'] + merged_at = pr_data["merged_at"] + author = pr_data["user"]["login"] if merged_at is None: continue @@ -439,7 +444,7 @@ def _iterate_by_page(org_name, repo_name): _iterate_by_page(org_name, repo_name) # convert unix timestamp to Beijing datetime - bj_start_datetime = datetime.fromtimestamp(start_datetime.timestamp(), tz=pytz.timezone('Asia/Shanghai')) + bj_start_datetime = datetime.fromtimestamp(start_datetime.timestamp(), tz=pytz.timezone("Asia/Shanghai")) bj_start_datetime_str = datetime2str(bj_start_datetime) contribution_list = counter.to_sorted_list() @@ -452,7 +457,7 @@ def _iterate_by_page(org_name, repo_name): if len(author_list) > 0: xlabel = f"Number of Pull Requests (since {bj_start_datetime_str})" ylabel = "Contributor" - title = 'Active Contributor Leaderboard' + title = "Active Contributor Leaderboard" plot_bar_chart(num_commit_list, author_list, xlabel=xlabel, ylabel=ylabel, title=title, output_path=output_path) return True else: @@ -468,14 +473,14 @@ def upload_image_to_lark(lark_tenant_token: str, image_path: str) -> str: image_path (str): the path to the image to be uploaded """ url = "https://open.feishu.cn/open-apis/im/v1/images" - form = {'image_type': 'message', 'image': (open(image_path, 'rb'))} # 需要替换具体的path + form = {"image_type": "message", "image": (open(image_path, "rb"))} # 需要替换具体的path multi_form = MultipartEncoder(form) headers = { - 'Authorization': f'Bearer {lark_tenant_token}', ## 获取tenant_access_token, 需要替换为实际的token + "Authorization": f"Bearer {lark_tenant_token}", ## 获取tenant_access_token, 需要替换为实际的token } - headers['Content-Type'] = multi_form.content_type + headers["Content-Type"] = multi_form.content_type response = requests.request("POST", url, headers=headers, data=multi_form).json() - return response['data']['image_key'] + return response["data"]["image_key"] def generate_lark_tenant_access_token(app_id: str, app_secret: str) -> str: @@ -486,10 +491,10 @@ def generate_lark_tenant_access_token(app_id: str, app_secret: str) -> str: app_id (str): Lark app id app_secret (str): Lark app secret """ - url = 'https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal' - data = {'app_id': app_id, 'app_secret': app_secret} + url = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal" + data = {"app_id": app_id, "app_secret": app_secret} response = requests.post(url, json=data).json() - return response['tenant_access_token'] + return response["tenant_access_token"] def send_image_to_lark(image_key: str, webhook_url: str) -> None: @@ -516,10 +521,10 @@ def send_message_to_lark(message: str, webhook_url: str): requests.post(webhook_url, json=data) -if __name__ == '__main__': - GITHUB_TOKEN = os.environ['GITHUB_TOKEN'] - CONTRIBUTOR_IMAGE_PATH = 'contributor_leaderboard.png' - USER_ENGAGEMENT_IMAGE_PATH = 'engagement_leaderboard.png' +if __name__ == "__main__": + GITHUB_TOKEN = os.environ["GITHUB_TOKEN"] + CONTRIBUTOR_IMAGE_PATH = "contributor_leaderboard.png" + USER_ENGAGEMENT_IMAGE_PATH = "engagement_leaderboard.png" ORG_NAME = "hpcaitech" # get all open source repositories @@ -527,17 +532,19 @@ def send_message_to_lark(message: str, webhook_url: str): # generate images contrib_success = generate_contributor_leaderboard_image(GITHUB_TOKEN, ORG_NAME, REPO_LIST, CONTRIBUTOR_IMAGE_PATH) - engagement_success = generate_user_engagement_leaderboard_image(GITHUB_TOKEN, ORG_NAME, REPO_LIST, USER_ENGAGEMENT_IMAGE_PATH) + engagement_success = generate_user_engagement_leaderboard_image( + GITHUB_TOKEN, ORG_NAME, REPO_LIST, USER_ENGAGEMENT_IMAGE_PATH + ) # upload images - APP_ID = os.environ['LARK_APP_ID'] - APP_SECRET = os.environ['LARK_APP_SECRET'] + APP_ID = os.environ["LARK_APP_ID"] + APP_SECRET = os.environ["LARK_APP_SECRET"] LARK_TENANT_TOKEN = generate_lark_tenant_access_token(app_id=APP_ID, app_secret=APP_SECRET) contributor_image_key = upload_image_to_lark(LARK_TENANT_TOKEN, CONTRIBUTOR_IMAGE_PATH) user_engagement_image_key = upload_image_to_lark(LARK_TENANT_TOKEN, USER_ENGAGEMENT_IMAGE_PATH) # send message to lark - LARK_WEBHOOK_URL = os.environ['LARK_WEBHOOK_URL'] + LARK_WEBHOOK_URL = os.environ["LARK_WEBHOOK_URL"] message = """本周的社区榜单出炉啦！ 1. 开发贡献者榜单 2. 用户互动榜单 diff --git a/.github/workflows/scripts/generate_release_draft.py b/.github/workflows/scripts/generate_release_draft.py index dc592e4c977b..7374481005ef 100644 --- a/.github/workflows/scripts/generate_release_draft.py +++ b/.github/workflows/scripts/generate_release_draft.py @@ -7,27 +7,27 @@ import requests -COMMIT_API = 'https://api.github.com/repos/hpcaitech/ColossalAI/commits' -TAGS_API = 'https://api.github.com/repos/hpcaitech/ColossalAI/tags' +COMMIT_API = "https://api.github.com/repos/hpcaitech/ColossalAI/commits" +TAGS_API = "https://api.github.com/repos/hpcaitech/ColossalAI/tags" def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('--out', type=str, help='output path for the release draft', required=True) - parser.add_argument('--version', type=str, help='current version to release', required=True) + parser.add_argument("--out", type=str, help="output path for the release draft", required=True) + parser.add_argument("--version", type=str, help="current version to release", required=True) return parser.parse_args() def get_latest_tag_commit(headers=None): res = requests.get(url=TAGS_API, headers=headers) data = res.json() - commit_hash = data[0]['commit']['sha'] - version = data[0]['name'] + commit_hash = data[0]["commit"]["sha"] + version = data[0]["name"] return commit_hash, version def get_commit_info(commit_hash, headers=None): - api = f'{COMMIT_API}/{commit_hash}' + api = f"{COMMIT_API}/{commit_hash}" res = requests.get(url=api, headers=headers) return res.json() @@ -37,7 +37,7 @@ def get_all_commit_info(since, headers=None): results = [] while True: - api = f'{COMMIT_API}?since={since}&per_page=100&page={page}' + api = f"{COMMIT_API}?since={since}&per_page=100&page={page}" resp = requests.get(url=api, headers=headers) data = resp.json() @@ -53,21 +53,21 @@ def get_all_commit_info(since, headers=None): def collate_release_info(commit_info_list): results = dict() - pattern = pattern = r'\[.*\]' + pattern = pattern = r"\[.*\]" for commit_info in commit_info_list: - author = commit_info['commit']['author']['name'] + author = commit_info["commit"]["author"]["name"] try: - author_url = commit_info['author']['url'] + author_url = commit_info["author"]["url"] except: # author can be None author_url = None - msg = commit_info['commit']['message'] + msg = commit_info["commit"]["message"] match = re.search(pattern, msg) if match: - tag = match.group().lstrip('[').rstrip(']').capitalize() + tag = match.group().lstrip("[").rstrip("]").capitalize() if tag not in results: results[tag] = [] results[tag].append((msg, author, author_url)) @@ -89,42 +89,43 @@ def generate_release_post_markdown(current_version, last_version, release_info): for msg, author, author_url in v: # only keep the first line - msg = msg.split('\n')[0] + msg = msg.split("\n")[0] if author_url: - item = f'{msg} by [{author}]({author_url})\n' + item = f"{msg} by [{author}]({author_url})\n" else: - item = f'{msg} by {author}\n' - text.append(f'- {item}') + item = f"{msg} by {author}\n" + text.append(f"- {item}") - text.append('\n') + text.append("\n") # add full change log text.append( - f'**Full Changelog**: https://github.com/hpcaitech/ColossalAI/compare/{current_version}...{last_version}') + f"**Full Changelog**: https://github.com/hpcaitech/ColossalAI/compare/{current_version}...{last_version}" + ) return text -if __name__ == '__main__': +if __name__ == "__main__": args = parse_args() - token = os.environ['GITHUB_API_TOKEN'] - headers = {'Authorization': token} + token = os.environ["GITHUB_API_TOKEN"] + headers = {"Authorization": token} # get previous release tag last_release_commit, last_version = get_latest_tag_commit(headers) last_release_commit_info = get_commit_info(last_release_commit, headers=headers) - last_release_date = last_release_commit_info['commit']['author']['date'] + last_release_date = last_release_commit_info["commit"]["author"]["date"] # get the commits since last release commit_info = get_all_commit_info(since=last_release_date, headers=headers) - commit_info = commit_info[:-1] # remove the release commit + commit_info = commit_info[:-1] # remove the release commit # collate into markdown release_info = collate_release_info(commit_info) markdown_text = generate_release_post_markdown(args.version, last_version, release_info) # write into a file - with open(args.out, 'w') as f: + with open(args.out, "w") as f: for line in markdown_text: f.write(line) diff --git a/.github/workflows/scripts/send_message_to_lark.py b/.github/workflows/scripts/send_message_to_lark.py index a113327a786e..bc005d93c3f5 100644 --- a/.github/workflows/scripts/send_message_to_lark.py +++ b/.github/workflows/scripts/send_message_to_lark.py @@ -5,8 +5,8 @@ def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument('-m', '--message', type=str) - parser.add_argument('-u', '--url', type=str) + parser.add_argument("-m", "--message", type=str) + parser.add_argument("-u", "--url", type=str) return parser.parse_args() @@ -15,6 +15,6 @@ def send_message_to_lark(message, webhook_url): requests.post(webhook_url, json=data) -if __name__ == '__main__': +if __name__ == "__main__": args = parse_args() send_message_to_lark(args.message, args.url) diff --git a/.isort.cfg b/.isort.cfg index 090aa28e39f3..ccbf575fdbfa 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -3,3 +3,5 @@ line_length = 120 multi_line_output=3 include_trailing_comma = true ignore_comments = true +profile = black +honor_noqa = true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 725d266375ef..9871e1184462 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,23 +1,31 @@ repos: + - repo: https://github.com/PyCQA/autoflake + rev: v2.2.1 + hooks: + - id: autoflake + name: autoflake (python) + args: ['--in-place', '--remove-unused-variables', '--remove-all-unused-imports', '--ignore-init-module-imports'] + - repo: https://github.com/pycqa/isort rev: 5.12.0 hooks: - id: isort name: sort all imports (python) - - repo: https://github.com/pre-commit/mirrors-yapf - rev: v0.32.0 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 23.9.1 hooks: - - id: yapf - name: yapf formatter - args: ['--style=.style.yapf', '--parallel', '--in-place'] + - id: black + name: black formatter + args: ['--line-length=120', '--target-version=py37', '--target-version=py38', '--target-version=py39','--target-version=py310'] - repo: https://github.com/pre-commit/mirrors-clang-format rev: v13.0.1 hooks: - id: clang-format name: clang formatter + types_or: [c++, c] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index 05be0dc6a3a5..000000000000 --- a/.style.yapf +++ /dev/null @@ -1,5 +0,0 @@ -[style] -based_on_style = google -spaces_before_comment = 4 -split_before_logical_operator = true -column_limit = 120 diff --git a/LICENSE b/LICENSE index 06629068faa5..59d456c5b8a1 100644 --- a/LICENSE +++ b/LICENSE @@ -428,3 +428,52 @@ Copyright 2021- HPC-AI Technology Inc. All rights reserved. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + ---------------- LICENSE FOR AutoGPTQ ---------------- + + From AutoGPTQ: + + MIT License + + Copyright (c) 2023 潘其威(William) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. + + ---------------- LICENSE FOR exllama ---------------- + + From exllama: + + MIT License + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/README.md b/README.md index 0ddcdab741a4..b2efb7910489 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ [![Documentation](https://readthedocs.org/projects/colossalai/badge/?version=latest)](https://colossalai.readthedocs.io/en/latest/?badge=latest) [![CodeFactor](https://www.codefactor.io/repository/github/hpcaitech/colossalai/badge)](https://www.codefactor.io/repository/github/hpcaitech/colossalai) [![HuggingFace badge](https://img.shields.io/badge/%F0%9F%A4%97HuggingFace-Join-yellow)](https://huggingface.co/hpcai-tech) - [![slack badge](https://img.shields.io/badge/Slack-join-blueviolet?logo=slack&)](https://join.slack.com/t/colossalaiworkspace/shared_invite/zt-z7b26eeb-CBp7jouvu~r0~lcFzX832w) + [![slack badge](https://img.shields.io/badge/Slack-join-blueviolet?logo=slack&)](https://github.com/hpcaitech/public_assets/tree/main/colossalai/contact/slack) [![WeChat badge](https://img.shields.io/badge/微信-加入-green?logo=wechat&)](https://raw.githubusercontent.com/hpcaitech/public_assets/main/colossalai/img/WeChat.png) @@ -25,6 +25,7 @@ ## Latest News +* [2023/09] [One Half-Day of Training Using a Few Hundred Dollars Yields Similar Results to Mainstream Large Models, Open-Source and Commercial-Free Domain-Specific Llm Solution](https://www.hpc-ai.tech/blog/one-half-day-of-training-using-a-few-hundred-dollars-yields-similar-results-to-mainstream-large-models-open-source-and-commercial-free-domain-specific-llm-solution) * [2023/09] [70 Billion Parameter LLaMA2 Model Training Accelerated by 195%](https://www.hpc-ai.tech/blog/70b-llama2-training) * [2023/07] [HPC-AI Tech Raises 22 Million USD in Series A Funding](https://www.hpc-ai.tech/blog/hpc-ai-tech-raises-22-million-usd-in-series-a-funding-to-fuel-team-expansion-and-business-growth) * [2023/07] [65B Model Pretraining Accelerated by 38%, Best Practices for Building LLaMA-Like Base Models Open-Source](https://www.hpc-ai.tech/blog/large-model-pretraining) @@ -33,8 +34,6 @@ * [2023/03] [AWS and Google Fund Colossal-AI with Startup Cloud Programs](https://www.hpc-ai.tech/blog/aws-and-google-fund-colossal-ai-with-startup-cloud-programs) * [2023/02] [Open Source Solution Replicates ChatGPT Training Process! Ready to go with only 1.6GB GPU Memory](https://www.hpc-ai.tech/blog/colossal-ai-chatgpt) * [2023/01] [Hardware Savings Up to 46 Times for AIGC and Automatic Parallelism](https://medium.com/pytorch/latest-colossal-ai-boasts-novel-automatic-parallelism-and-offers-savings-up-to-46x-for-stable-1453b48f3f02) -* [2022/11] [Diffusion Pretraining and Hardware Fine-Tuning Can Be Almost 7X Cheaper](https://www.hpc-ai.tech/blog/diffusion-pretraining-and-hardware-fine-tuning-can-be-almost-7x-cheaper) -* [2022/10] [Use a Laptop to Analyze 90% of Proteins, With a Single-GPU Inference Sequence Exceeding 10,000](https://www.hpc-ai.tech/blog/use-a-laptop-to-analyze-90-of-proteins-with-a-single-gpu-inference-sequence-exceeding) ## Table of Contents