Use forks only on Request

.github/workflows/build-and-push.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           dockerfiles: ./Dockerfile
           image: betka
-          tags: latest 1 ${{ github.sha }} 0.8.1
+          tags: latest 1 ${{ github.sha }} 0.9.0
 
       - name: Push betka image to Quay.io
         id: push-to-quay

Dockerfile

@@ -1,7 +1,7 @@
 FROM quay.io/fedora/fedora:37
 
 ENV NAME=betka-fedora \
-    RELEASE=0.8.1 \
+    RELEASE=0.9.0
     ARCH=x86_64 \
     SUMMARY="Syncs changes from upstream repository to downstream" \
     DESCRIPTION="Syncs changes from upstream repository to downstream" \

betka/core.py

@@ -50,7 +50,7 @@
     SYNC_INTERVAL,
 )
 from betka.utils import FileUtils
-from betka.named_tuples import ProjectMR, ProjectFork
+from betka.named_tuples import ProjectMR, ProjectFork, ProjectInfo
 
 
 requests.packages.urllib3.disable_warnings()
@@ -78,6 +78,7 @@ def __init__(self, task_name=None):
         self.upstream_hash = None
         self.downstream_dir: Path = None
         self.downstream_git_branch: str = None
+        self.downstream_git_origin_branch: str = None
         self.repo = None
         self.pr_number = None
         self._github_api = self._gitlab_api = None
@@ -99,6 +100,7 @@ def set_config(self):
         """
         self.set_config_from_env(self.config_json["github_api_token"])
         self.set_config_from_env(self.config_json["gitlab_user"])
+        self.set_config_from_env(self.config_json["use_gitlab_forks"])
         if "slack_webhook_url" in self.config_json:
             self.set_config_from_env(self.config_json["slack_webhook_url"])
         self.set_config_from_env("PROJECT")
@@ -201,6 +203,12 @@ def prepare_upstream_git(self):
         self.info("Upstream cloned directory %r", self.upstream_cloned_dir)
         return True
 
+    def is_fork_enabled(self):
+        value = self.betka_config["use_gitlab_forks"].lower()
+        if value in ["true", "yes"]:
+            return True
+        return False
+
     def mandatory_variables_set(self):
         """
         Are mandatory default 'betka' values set ?
@@ -323,7 +331,7 @@ def send_result_email(self, betka_schema: Dict):
         )
         return True
 
-    def sync_to_downstream_branches(self, branch):
+    def sync_to_downstream_branches(self, branch, origin_branch: str = ""):
         """
         Sync upstream repository into relevant downstream dist-git branch
         based on the configuration file.
@@ -337,7 +345,7 @@ def sync_to_downstream_branches(self, branch):
             hash=self.upstream_hash, repo=self.repo
         )
         mr = self.gitlab_api.check_gitlab_merge_requests(branch=branch)
-        if not mr:
+        if not mr and self.is_fork_enabled():
             Git.get_changes_from_distgit(url=self.gitlab_api.get_forked_ssh_url_to_repo())
             Git.push_changes_to_fork(branch=branch)
 
@@ -361,6 +369,7 @@ def sync_to_downstream_branches(self, branch):
             upstream_hash=self.upstream_hash,
             branch=branch,
             mr=mr,
+            origin_branch=origin_branch,
         )
         self.send_result_email(betka_schema=betka_schema)
 
@@ -493,7 +502,7 @@ def prepare(self):
             return False
         return True
 
-    def prepare_downstream_git(self, project_fork: ProjectFork) -> bool:
+    def prepare_fork_downstream_git(self, project_fork: ProjectFork) -> bool:
 
         """
         Clone downstream dist-git repository, defined by self.ssh_url_to_repo variable
@@ -514,6 +523,26 @@ def prepare_downstream_git(self, project_fork: ProjectFork) -> bool:
 
         return True
 
+    def prepare_downstream_git(self, project_info: ProjectInfo) -> bool:
+
+        """
+        Clone downstream dist-git repository, defined by self.ssh_url_to_repo variable
+        and set `self.downstream_dir` variable.
+        :returns True if downstream git directory was cloned
+                 False if downstream git directory was not cloned
+        """
+        self.downstream_dir = Git.clone_repo(
+            project_info.ssh_url_to_repo, self.betka_tmp_dir.name
+        )
+        self.info("Downstream directory %r", self.downstream_dir)
+        if self.downstream_dir is None:
+            self.error("!!!! Cloning downstream repo %s FAILED.", self.image)
+            return False
+        os.chdir(str(self.downstream_dir))
+        # This function updates fork based on the upstream
+
+        return True
+
     def _copy_cloned_upstream_dir(self):
         """
         Copy cloned upstream directory stored in self.upstream_cloned_dir
@@ -591,16 +620,20 @@ def _update_valid_branches(self):
         Git.sync_fork_with_upstream(branch_list_to_sync)
         return branch_list_to_sync
 
-    def _update_valid_remote_branches(self):
-        # Branches are taken from upstream repository like
-        # https://src.fedoraproject.org/container/nginx not from fork
+    def _get_valid_origin_branches(self):
         all_branches = Git.get_valid_remote_branches()
         self.debug(f"All remote branches {all_branches}.")
         # Filter our branches before checking bot-cfg.yml files
         branch_list_to_sync = Git.branches_to_synchronize(
             self.betka_config, all_branches=all_branches
         )
         self.debug(f"Branches to sync {branch_list_to_sync}")
+        return branch_list_to_sync
+
+    def _update_valid_remote_branches(self):
+        # Branches are taken from upstream repository like
+        # https://src.fedoraproject.org/container/nginx not from fork
+        branch_list_to_sync = self._get_valid_origin_branches()
         Git.sync_fork_with_upstream(branch_list_to_sync)
         return branch_list_to_sync
 
@@ -617,7 +650,12 @@ def _sync_valid_branches(self, valid_branches):
             raise
         for branch in valid_branches:
             self.timestamp_dir: Path = None
-            self.downstream_git_branch = branch
+            if self.is_fork_enabled():
+                self.downstream_git_branch = branch
+                self.downstream_git_origin_branch = ""
+            else:
+                self.downstream_git_branch = f"betka-{branch}"
+                self.downstream_git_origin_branch = branch
             # This loads downstream bot-cfg.yml file
             # and update betka's dictionary (self.config).
             # We need to have information up to date
@@ -630,7 +668,7 @@ def _sync_valid_branches(self, valid_branches):
             if not self.config.get("master_checker"):
                 continue
             self.create_and_copy_timestamp_dir()
-            self.sync_to_downstream_branches(self.downstream_git_branch)
+            self.sync_to_downstream_branches(self.downstream_git_branch, self.downstream_git_origin_branch)
             self.delete_timestamp_dir()
 
     def run_sync(self):
@@ -659,7 +697,7 @@ def _run_sync(self):
             # variable dist_git_repos
 
             try:
-                self.gitlab_api.get_project_id_from_url()
+                project_id = self.gitlab_api.get_project_id_from_url()
             except requests.exceptions.HTTPError as htpe:
                 BetkaEmails.send_email(
                     text=f"Get project from URL {self.image} were not successful"
@@ -669,31 +707,35 @@ def _run_sync(self):
                     subject=f"[betka-sync] Get project from URL project {self.image} were not successful.",
                 )
                 continue
-            self.gitlab_api.init_projects()
-            project_fork = self.gitlab_api.check_and_create_fork()
-            if not project_fork:
-                BetkaEmails.send_email(
-                    text=f"Fork for project {self.image} were not successful"
-                    f"by upstream2downstream-bot. See {values}\n"
-                    f"Inform phracek@redhat.com",
-                    receivers=["phracek@redhat.com"],
-                    subject=f"[betka-sync] Fork for project {self.image} were not successful.",
-                )
-                continue
-
+            if self.is_fork_enabled():
+                self.gitlab_api.init_projects()
+                project_fork = self.gitlab_api.check_and_create_fork()
+                if not project_fork:
+                    BetkaEmails.send_email(
+                        text=f"Fork for project {self.image} were not successful"
+                        f"by upstream2downstream-bot. See {values}\n"
+                        f"Inform phracek@redhat.com",
+                        receivers=["phracek@redhat.com"],
+                        subject=f"[betka-sync] Fork for project {self.image} were not successful.",
+                    )
+                    continue
+                self.ssh_url_to_repo = project_fork.ssh_url_to_repo
+                self.debug(f"Clone URL is: {self.ssh_url_to_repo}")
+                os.chdir(self.betka_tmp_dir.name)
+                if not self.prepare_fork_downstream_git(project_fork):
+                    continue
+                branch_list_to_sync = self._update_valid_remote_branches()
+            else:
+                project_info = self.gitlab_api.get_project_info()
+                self.ssh_url_to_repo = project_info.ssh_url_to_repo
+                self.debug(f"Clone URL is: {self.ssh_url_to_repo}")
+                os.chdir(self.betka_tmp_dir.name)
+                if not self.prepare_downstream_git(project_info):
+                    continue
+                branch_list_to_sync = self._get_valid_origin_branches()
             self.info(
                 f"Trying to sync image {self.image} to GitLab project_id {self.gitlab_api.project_id}."
             )
-            os.chdir(self.betka_tmp_dir.name)
-
-            self.ssh_url_to_repo = project_fork.ssh_url_to_repo
-            self.debug(f"Clone URL is: {self.ssh_url_to_repo}")
-            # after downstream is cloned then
-            # new cwd is self.downstream_dir
-            if not self.prepare_downstream_git(project_fork):
-                continue
-            #branch_list_to_sync = self._update_valid_branches()
-            branch_list_to_sync = self._update_valid_remote_branches()
 
             valid_branches = Git.get_valid_branches(
                 self.image, self.downstream_dir, branch_list_to_sync

betka/gitlab.py

@@ -305,13 +305,15 @@ def file_merge_request(
         pr_msg: str,
         upstream_hash: str,
         branch: str,
+        origin_branch: str,
         mr: Any,
     ) -> Dict:
         """
         Files a Pull Request with specific messages and text.
         :param pr_msg: description message used in pull request
         :param upstream_hash: commit hash for
         :param branch: specify downstream branch for file a Pull Request
+        :param origin_branch: specify origin_branch to which file a Pull Request
         :param mr: named touple ProjectMR
         :return: schema for sending email
         """
@@ -323,7 +325,7 @@ def file_merge_request(
             logger.debug(f"Upstream {text_mr} to downstream PR not found.")
 
             mr: ProjectMR = self.create_gitlab_merge_request(
-                title=title, desc_msg=pr_msg, branch=branch
+                title=title, desc_msg=pr_msg, branch=branch, origin_branch=origin_branch
             )
             logger.debug(f"MergeRequest is: {mr}")
             if mr is None:
@@ -375,7 +377,7 @@ def init_projects(self) -> bool:
         return True
 
     def create_gitlab_merge_request(
-        self, title: str, desc_msg: str, branch: str
+        self, title: str, desc_msg: str, branch: str, origin_branch: str,
     ) -> ProjectMR:
         """
         Creates the pull request for specific image
@@ -392,6 +394,8 @@ def create_gitlab_merge_request(
             "description": desc_msg,
             "target_project_id": self.project_id,
         }
+        if not self.betka_config["use_gitlab_fork"]:
+            data["target_branch"] = origin_branch
         return self.create_project_mergerequest(data)
 
     def check_gitlab_merge_requests(self, branch: str):
@@ -519,3 +523,4 @@ def get_project_id_from_url(self):
             raise HTTPError
         self.project_id = ret.json()["id"]
         logger.debug(f"Project id returned from {url} is {self.project_id}")
+        return self.project_id

config.json

@@ -18,5 +18,6 @@
   "gitlab_host_url": "https://gitlab.com",
   "gitlab_namespace": "redhat/rhel/containers",
   "dist_git_url": "https://src.fedoraproject.org/container",
-  "slack_webhook_url": "SLACK_WEBHOOK_URL"
+  "slack_webhook_url": "SLACK_WEBHOOK_URL",
+  "use_gitlab_forks": "False"
 }

setup.py

@@ -14,7 +14,7 @@ def get_requirements():
 
 setup(
     name="betka",
-    version="0.8.1",
+    version="0.9.0",
     packages=find_packages(exclude=["examples", "tests"]),
     url="https://github.com/sclorg/betka",
     license="GPLv3+",

tests/conftest.py

@@ -57,6 +57,7 @@ def betka_yaml():
             },
         },
         "downstream_master_msg": "[betka-master-sync]",
+        "use_gitlab_forks": "True"
     }
 
 
@@ -84,6 +85,7 @@ def config_json():
         "gitlab_url_user": "user",
         "dist_git_url": "https://src.fedoraproject.org/containers",
         "slack_webhook_url": "SLACK_WEBHOOK_URL",
+        "use_gitlab_forks": "True",
     }
 
 

tests/integration/test_betka_upstream_sync.py

@@ -93,6 +93,7 @@ def setup_method(self):
         os.environ["GITLAB_API_TOKEN"] = "gitlabsomething"
         os.environ["GITHUB_API_TOKEN"] = "aklsdjfh19p3845yrp"
         os.environ["GITLAB_USER"] = "testymctestface"
+        os.environ["USE_GITLAB_FORKS"] = "true"
         flexmock(FileUtils).should_receive("load_config_json").and_return(config_json())
         self.betka = Betka(task_name="task.betka.master_sync")
         self.config_json = config_json()
@@ -239,6 +240,7 @@ def test_betka_run_master_sync(
         mock_rmtree,
     ):
         self.betka.betka_config["dist_git_repos"].pop("s2i-core")
+        self.betka.betka_config["use_gitlab_forks"] = "True"
         list_images = self.betka.get_synced_images()
         sync_image = ""
         for key, value in list_images.items():
@@ -251,6 +253,7 @@ def test_betka_run_master_sync(
         flexmock(self.betka).should_receive("_update_valid_remote_branches").and_return(
             ["fc30", "fc31"]
         )
+        flexmock(self.betka).should_receive("prepare_fork_downstream_git").twice()
         # flexmock(Git).should_receive("sync_fork_with_upstream").twice()
         self.betka.gitlab_api.project_id = PROJECT_ID
         flexmock(self.betka.gitlab_api).should_receive("get_project_id_from_url").and_return(PROJECT_ID)

tests/unit/test_betka_config_json.py

@@ -64,6 +64,19 @@ def config_json_missing_generator_url():
     }
 
 
+def config_json_missing_use_gitlab_fork():
+    return {
+        "api_url": "https://src.fedoraproject.org/api/0",
+        "get_all_pr": "https://src.fedoraproject.org/api/0/{namespace}/{repo}/pull-requests",
+        "git_url_repo": "https://src.fedoraproject.org/api/0/fork/{user}/{namespace}/{repo}/git/",
+        "namespace_containers": "container",
+        "github_api_token": "aklsdjfh19p3845yrp",
+        "gitlab_user": "testymctestface",
+        "gitlab_api_token": "testing",
+        "generator_url": "some_generator_url",
+    }
+
+
 class TestBetkaCore(object):
     def setup_method(self):
         self.betka = Betka()
@@ -73,6 +86,7 @@ def setup_method(self):
         [
             config_json_missing_github_api_token(),
             config_json_missing_generator_url(),
+            config_json_missing_use_gitlab_fork(),
         ],
     )
     def test_betka_config_keyerror(self, config_json):

tests/unit/test_core.py

@@ -85,6 +85,19 @@ def test_get_synced_images(self, msg_upstream_url, return_value):
         for image, values in dict_images.items():
             assert image == image in return_value
 
+    @pytest.mark.parametrize(
+        "status,return_value",
+        [
+            ("True", True),
+            ("true", True),
+            ("False", False),
+            ("false", False),
+        ],
+    )
+    def test_is_fork_enabled(self, status, return_value):
+        self.betka.betka_config["use_gitlab_forks"] = status
+        assert self.betka.is_fork_enabled() == return_value
+
     @pytest.mark.parametrize(
         "msg_upstream_url,return_value",
         [