Merge develop -> main

automation/scripts/github_branch_autosync/github_autosync/__main__.py

@@ -7,16 +7,26 @@
 from gcloud_entrypoint import handle_incoming_commit_push_json,config,verify_signature
 
 parser = argparse.ArgumentParser()
-parser.add_argument('payload', help='test file from github webhook push event')
-parser.add_argument('secret', help='secret for calculating signature')
-parser.add_argument('incoming_signature', help='payload signature')
+parser.add_argument('--operation', "-o",type=str, help='debug operation to perform',required=True)
+parser.add_argument('--payload', "-p",type=str, help='test file from github webhook push event',required=False)
+parser.add_argument('--secret', "-s", type=str, help='secret for calculating signature',required=False)
+parser.add_argument('--incoming_signature', "-i",type=str,  help='payload signature',required=False)
 
 args = parser.parse_args()
 
-if not os.path.isfile(args.payload):
-    sys.exit('cannot find test file :',args.payload)
+if "verify" in args.operation:
+    if not os.path.isfile(args.payload):
+        sys.exit('cannot find test file :',args.payload)
 
-with open(args.payload,encoding="utf-8") as file:
-    data = json.load(file)
-    json_payload = json.dumps(data)
-    verify_signature(json_payload, args.secret, "sha=" + args.incoming_signature)
+    with open(args.payload,encoding="utf-8") as file:
+        data = json.load(file)
+        json_payload = json.dumps(data)
+        verify_signature(json_payload, args.secret, "sha=" + args.incoming_signature)
+
+elif "handle_payload" in args.operation:
+    with open(args.payload,encoding="utf-8") as file:
+        data = json.load(file)
+        json_payload = json.dumps(data)
+        handle_incoming_commit_push_json(data,config=config)
+else:
+    print("operation no supported", file=sys.stderr)

automation/scripts/github_branch_autosync/github_autosync/gcloud_entrypoint/lib/config.py

@@ -13,8 +13,7 @@
 program will attempt to merge new changes to compatible branch
 '''
 branches = dict(
-    compatible = 'rampup',
-    rampup = 'berkeley',
+    compatible = 'berkeley',
     berkeley = 'develop'
 )
 
@@ -47,17 +46,21 @@ def tmp_branch_name(source_branch,target_branch):
     '''
         Method which will be used for naming temp branch (needed for checking merge ability)
     '''
-    return f"sync-{source_branch}-with-{target_branch}"
+    return f"fix-conflict-of-{source_branch}-and-{target_branch}"
 
 '''
 Specific settings for PR creation (if there is necessity to do it based on current repo situation).
 '''
 pr = {
-    "title_prefix": "[Branches auto sync failure] ",
+    "title_prefix": "[Fix me] Merge conflict between ",
     "assignees": ["dkijania"],
     "body_prefix": "This is auto-generated PR in order to solve merge conflicts between two branches.",
     "draft": 'false',
-    "labels": ["auto-sync"]
+    "labels": ["auto-sync"],
+    "alert_header": """
+# :exclamation: New Conflict detected :exclamation: 
+This PR conflicts with one of our main branches. As a result below Pull requests were created to aid you in resolving merge conflicts. Each temporary branch contains *cherry picked* changes from this PR. 
+"""
 }
 
 '''

automation/scripts/github_branch_autosync/github_autosync/gcloud_entrypoint/lib/github.py

@@ -1,7 +1,7 @@
 ''' Github api tailored for auto-sync needs'''
 
 import json
-from github import Github,PullRequest
+from github import Github,PullRequest,InputGitTreeElement
 import requests
 
 class GithubException(Exception):
@@ -145,6 +145,42 @@ def delete_branch(self, branch_name):
         '''
         self.repository().delete_branch(branch_name)
 
+    def cherry_pick_commits(self,new_branch,commits,skip_merges):
+        '''
+           Cherry picks commits to new branch. It doesn't perform true git cherry pick 
+           but rather manually copies git tree and along with commit messages and applies 
+           it to new base tree
+
+            Parameters:
+                new_branch (string): Branch name to insert commits
+                commits (List of GitCommit): List of commits to apply
+                skip_merges (Bool): Flag which controls if we should apply merge commits
+        '''
+        if skip_merges:
+            commits = list(filter(lambda commit: len(commit.parents) < 2, commits))            
+
+        for commit in commits:
+            template_tree = self.repository().inner.get_git_tree(commit.sha)
+
+            branch_obj = self.repository().inner.get_branch(new_branch)
+            base_tree = self.repository().inner.get_git_tree(branch_obj.commit.sha)
+
+
+            inputs = []
+            for element in template_tree.tree:
+                inputs.append(InputGitTreeElement(
+                    path=element.path, mode=element.mode, type=element.type, sha=element.sha
+                ))
+
+            tree = self.repository().inner.create_git_tree(inputs, base_tree)
+            commit = self.repository().inner.create_git_commit(
+                message=commit.commit.message,
+                tree=tree,
+                parents=[branch_obj.commit.commit]
+            )
+
+        self.repository().update_ref(new_branch,commit.sha)
+
     def create_pull_request(self,config,source_branch,target_branch,new_branch):
         """ 
             Creates new pull request
@@ -158,7 +194,7 @@ def create_pull_request(self,config,source_branch,target_branch,new_branch):
             Returns:
                 return PullRequest object
         """
-        title = config.pr["title_prefix"] + f"into {source_branch} from {target_branch}"
+        title = config.pr["title_prefix"] + f" {source_branch} and {target_branch}"
         assignee_tags = list(map(lambda x: "@" + x, config.pr["assignees"]))
         separator = ", "
         body = config.pr["body_prefix"] + "\n" + separator.join(assignee_tags)
@@ -181,7 +217,7 @@ def create_pull_request_for_tmp_branch(self,config,source_branch,temp_branch):
             Returns:
                 return PullRequest object
         """
-        title = config.pr["title_prefix"] + f"into {source_branch} from {temp_branch} for commit {self.branch(source_branch).commit.sha[0:6]}"
+        title = config.pr["title_prefix"] + f"{source_branch} from {temp_branch} for commit {self.branch(source_branch).commit.sha[0:6]}"
         assignee_tags = list(map(lambda x: "@" + x, config.pr["assignees"]))
         separator = ", "
         body = config.pr["body_prefix"] + "\n" + separator.join(assignee_tags)
@@ -207,10 +243,19 @@ def merge(self,base,head,message):
             Parameters:
                 base (string): base branch name
                 head (string): head branch name
-                commit (string): commit message
+                commit (string): commit message 
 
         """
         self.repository().merge(base,head,message)
+
+    def get_opened_not_draft_prs_from(self,head):
+        """
+            Get prs with given head which are non draft and opened
+
+            Parameters:
+                head (string): head branch name
+        """
+        return list(self.repository().get_pulls_from(head,draft=False,open=True))
 
 class Repository:
     """
@@ -237,6 +282,18 @@ def merge(self,base,head,message):
         else:
             self.inner.merge(base,head,message)
 
+    def get_pulls_from(self,head,open,draft):
+        """
+            Get prs with given head and  state
+
+            Parameters:
+                head (string): head branch name
+                open (Bool): is opened
+                draft (Bool): is draft PR             
+        """
+        state = "open" if open else "closed"
+        return filter(lambda x: x.draft == draft and x.head.ref == head, self.inner.get_pulls(state,head))
+
     def create_pull(self,title,body,base,head,draft,assignees,labels):
         if self.dryrun:
             print(f'{self.dryrun_suffix} Pull request created:')
@@ -285,6 +342,28 @@ def fast_forward(self,source,target_sha):
             return output["object"]["sha"]
         raise GithubException(f'unable to fast forward branch {source} due to : {res.text}')
 
+    def update_ref(self,source,target_sha):
+        """
+            Force update ref to new sha
+
+            Parameters:
+                source (string): source branch name
+                target_sha (Bool): target ref         
+        """
+        if self.dryrun:
+            print(f"{self.dryrun_suffix} Updating ref '{source}' to '{target_sha}'")
+            return target_sha
+        res = requests.patch(f"https://api.github.com/repos/{self.username}/{self.repo}/git/refs/heads/{source}",
+                        json={"sha": target_sha, "force": True},
+                        headers=self.authorization_header,
+                        timeout=self.timeout
+                        )
+        if res.status_code == 200:
+            output =  json.loads(res.text)
+            return output["object"]["sha"]
+        raise GithubException(f'unable to fast forward branch {source} due to : {res.text}')
+
+
     def delete_branch(self,branch_name):
         if self.dryrun:
             print(f"{self.dryrun_suffix} Delete branch '{branch_name}'")

automation/scripts/github_branch_autosync/github_autosync/gcloud_entrypoint/main.py

@@ -20,17 +20,11 @@ def handle_incoming_commit_push(request):
     print("done")
     return
 
-def handle_incoming_commit_push_json(json,config):
-    """
-      Main logic for handling incoming github webhook event
+def handle_incoming_commit_push_in_stable_branches(source_branch):
+    """Hand incoming commit on major branch.
+    Args:
+        source_branch (String): Name of branch which commit was pushed to.
     """
-    payload_info=  GithubPayloadInfo(json)
-
-    source_branch = payload_info.incoming_branch
-
-    if not source_branch in config.branches:
-        print(f"change in '{source_branch}' is not supported ")
-        return
 
     target_branch = config.branches[source_branch]
     github = GithubApi(config.github)
@@ -78,3 +72,120 @@ def handle_incoming_commit_push_json(json,config):
             print(f"there is no merge conflict. merging {new_branch} into {target_branch}...")
             github.merge(target_branch,new_branch, f"Github Autosync: {source_branch} -> {target_branch}")
             github.delete_branch(new_branch)
+
+def get_branches_earlier_in_chain(branches,branch):
+    """ Retrieves names of branches earlier in the chain that incoming one
+    Args:
+        branches (Dictionary): Configuration element which defines branches relation.
+        branch (String): Incoming branch
+    Returns:
+        List of branches earlier in the chain
+    """
+    inv_branches = {v: k for k, v in branches.items()}
+    return get_branches_later_in_chain(inv_branches,branch)
+
+def get_branches_later_in_chain(branches,branch):
+    """ Retrieves names of branches earlier in the chain that incoming one
+    Args:
+        branches (Dictionary): Configuration element which defines branches relation.
+        branch (String): Incoming branch
+    Returns:
+        List of branches later in the chain
+    """
+    output = []
+    next = branches.get(branch)
+    while next is not None:
+        output.append(next)
+        next = branches.get(next)
+    return output
+
+
+def handle_pr(pr,github,source_branch):
+    """ Handle push in personal pr
+    Args:
+        pr (PullRequest): Configuration element which defines branches relation.
+        github (Github): Github wrapper
+        branch (String): Incoming branch
+    """
+    branches = get_branches_earlier_in_chain(config.branches,pr.base.ref)
+    later_branches = get_branches_later_in_chain(config.branches,pr.base.ref)
+    branches.extend(later_branches)
+
+    data = []
+
+    for branch in branches: 
+        if github.has_merge_conflict(branch,source_branch):
+            print(f"{branch} and {source_branch} branches have a merge conflict! creating PR to address those changes...")
+
+            new_branch = config.tmp_branch_name(source_branch,branch)
+
+            if github.branch_exists(new_branch):
+                print(f"{new_branch} already exists therefore we will recreate it")
+                github.delete_branch(new_branch)
+
+            github.create_new_branch(new_branch,branch)
+
+            commits = pr.get_commits()
+            github.cherry_pick_commits(new_branch,commits,skip_merges=True)
+
+            title = github.create_pull_request(config,source_branch,branch,new_branch)
+            print(f"new PR: '{title}' created. Please resolve it before merge...")
+
+            for pr in github.repository().inner.get_pulls(head=new_branch):
+                if pr.title == title:
+                    data.append((pr.html_url,new_branch,branch))
+    if any(data):
+        pr.create_issue_comment(comment_conflict(data))
+
+def handle_incoming_commit_push_in_personal_branches(source_branch):
+    """
+      Main handler for change in personal branch
+    """
+    github = GithubApi(config.github)
+
+    pull_requests = github.get_opened_not_draft_prs_from(source_branch)
+
+    if not any(pull_requests):
+        print(f"skipping... merge check as branch {source_branch} does not have any non-draft pr opened")
+
+    for pr in pull_requests:
+        handle_pr(pr,github,source_branch)
+
+def comment_conflict(data):
+    """
+      Template for issue comment after conflict in PR is detected
+    """
+    content = config.pr["alert_header"] + """
+<table>
+  <tr>
+    <th>Pull request name</th>
+    <th> Temporary branch name</th>
+    <th> Conflicting branch </th>
+  </tr>
+"""
+    for (url,base,branch) in data: 
+        content = content + f"""
+<tr>
+    <td> <a href="{url}"> {url} </a> </td>
+    <td> {base}  </td>
+    <td> {branch} </td>
+</tr>
+"""
+    content = content + """
+</table>
+"""
+    return content
+
+def handle_incoming_commit_push_json(json,config):
+    """
+      Main logic for handling incoming github webhook event
+    """
+    payload_info=  GithubPayloadInfo(json)
+
+    source_branch = payload_info.incoming_branch
+
+    if not source_branch in config.branches:
+        handle_incoming_commit_push_in_personal_branches(source_branch)
+    else:
+        handle_incoming_commit_push_in_stable_branches(source_branch)
+