Skip to content

Commit

Permalink
refactor: Primay Branch Handling
Browse files Browse the repository at this point in the history
Many assumptions around branches and repos were made early on, since then we moved
a lot of the common repo actions into relevant classes. This furthers that work and
replaces all direct branch access with methods that automatically figure out the
primary branch of the repo. Tests updated to reference 'main'
  • Loading branch information
techman83 committed Mar 27, 2023
1 parent 51471cc commit ee03890
Show file tree
Hide file tree
Showing 16 changed files with 342 additions and 184 deletions.
2 changes: 1 addition & 1 deletion netkan/netkan/auto_freezer.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ def __init__(self, nk_repo: NetkanRepo, github_pr: GitHubPR) -> None:
self.github_pr = github_pr

def freeze_idle_mods(self, days_limit: int, days_till_ignore: int) -> None:
self.nk_repo.git_repo.remotes.origin.pull('master', strategy_option='ours')
self.nk_repo.pull_remote_primary(strategy_option='ours')
idle_mods = self._find_idle_mods(days_limit, days_till_ignore)
if idle_mods:
with self.nk_repo.change_branch(self.BRANCH_NAME):
Expand Down
5 changes: 2 additions & 3 deletions netkan/netkan/cli/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
from typing import Union, Callable, Any, List, Optional, Tuple

import click
from git import Repo

from ..repos import NetkanRepo, CkanMetaRepo
from ..utils import init_repo, init_ssh
Expand Down Expand Up @@ -129,8 +128,8 @@ def netkan_repo(self) -> NetkanRepo:
return self._netkan_repo

@property
def repos(self) -> List[Repo]:
return [self.ckanmeta_repo.git_repo, self.netkan_repo.git_repo]
def repos(self) -> List[Union[NetkanRepo, CkanMetaRepo]]:
return [self.ckanmeta_repo, self.netkan_repo]

@property
def netkan_remote(self) -> str:
Expand Down
11 changes: 5 additions & 6 deletions netkan/netkan/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ def sqs_batch_entries(messages: Iterable[SendMessageBatchRequestEntryTypeDef],
yield batch


def pull_all(repos: Iterable[Repo]) -> None:
def pull_all(repos: Iterable[Union[NetkanRepo, CkanMetaRepo]]) -> None:
for repo in repos:
repo.remotes.origin.pull('master', strategy_option='theirs')
repo.pull_remote_primary(strategy_option='theirs')


def github_limit_remaining(token: str) -> int:
Expand Down Expand Up @@ -83,10 +83,9 @@ def __init__(self, game: Game) -> None:
# we can ensure we call close on it and run our handler inside
# a context manager
def __enter__(self) -> 'BaseMessageHandler':
if not self.repo.is_active_branch('master'):
self.repo.checkout_branch('master')
self.repo.pull_remote_branch(
'master', strategy_option=self.STRATEGY_OPTION)
if not self.repo.is_primary_active():
self.repo.checkout_primary()
self.repo.pull_remote_primary(strategy_option=self.STRATEGY_OPTION)
return self

def __exit__(self, exc_type: Type[BaseException],
Expand Down
13 changes: 6 additions & 7 deletions netkan/netkan/download_counter.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ def get_result(self, counts: Optional[Dict[str, int]] = None) -> Dict[str, int]:
result = self.graphql_to_github(full_query)
if 'errors' in result:
logging.error('DownloadCounter errors in GraphQL query: %s',
', '.join(err['message'] for err in result['errors'])
if result['errors'] else 'Empty errors list')
', '.join(err['message'] for err in result['errors'])
if result['errors'] else 'Empty errors list')
if 'data' in result:
for fake_ident, apidata in result['data'].items():
if apidata:
Expand All @@ -119,7 +119,8 @@ def get_result(self, counts: Optional[Dict[str, int]] = None) -> Dict[str, int]:
def graphql_to_github(self, query: str) -> Dict[str, Any]:
logging.info('Contacting GitHub')
return requests.post(self.GITHUB_API,
headers={'Authorization': f'bearer {self.github_token}'},
headers={
'Authorization': f'bearer {self.github_token}'},
json={'query': query},
timeout=60).json()

Expand Down Expand Up @@ -278,14 +279,12 @@ def commit_counts(self) -> None:
'NetKAN Updating Download Counts'
)
logging.info('Download counts changed and committed')
self.ckm_repo.git_repo.remotes.origin.push('master')
self.ckm_repo.push_remote_primary()

def update_counts(self) -> None:
if self.output_file:
self.get_counts()
self.ckm_repo.git_repo.remotes.origin.pull(
'master', strategy_option='ours'
)
self.ckm_repo.pull_remote_primary(strategy_option='ours')
self.write_json()
if repo_file_add_or_changed(self.ckm_repo.git_repo, self.output_file):
self.commit_counts()
Expand Down
7 changes: 6 additions & 1 deletion netkan/netkan/github_pr.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,14 @@ def repo(self) -> Repository:
f'{self.user}/{self.git_repo}')
return self._repo

@property
def default_branch(self) -> str:
return self.repo.default_branch

def create_pull_request(self, title: str, branch: str, body: str, labels: Optional[List[str]] = None) -> None:
try:
pull = self.repo.create_pull(title, body, 'master', branch)
pull = self.repo.create_pull(
title, body, self.default_branch, branch)
logging.info('Pull request for %s opened at %s',
branch, pull.html_url)

Expand Down
24 changes: 12 additions & 12 deletions netkan/netkan/indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -171,15 +171,15 @@ def delete_attrs(self) -> DeleteMessageBatchRequestEntryTypeDef:


class MessageHandler(BaseMessageHandler):
_master: Deque[CkanMessage]
_primary: Deque[CkanMessage]
_staged: Deque[CkanMessage]
_processed: List[CkanMessage]

def __str__(self) -> str:
return str(' '.join([str(x) for x in self.master + self.staged]))
return str(' '.join([str(x) for x in self.primary + self.staged]))

def __len__(self) -> int:
return len(self.master + self.staged)
return len(self.primary + self.staged)

@property
def repo(self) -> CkanMetaRepo:
Expand All @@ -190,10 +190,10 @@ def github_pr(self) -> GitHubPR:
return self.game.github_pr

@property
def master(self) -> Deque[CkanMessage]:
if getattr(self, '_master', None) is None:
self._master = deque()
return self._master
def primary(self) -> Deque[CkanMessage]:
if getattr(self, '_primary', None) is None:
self._primary = deque()
return self._primary

@property
def staged(self) -> Deque[CkanMessage]:
Expand All @@ -214,7 +214,7 @@ def append(self, message: Message) -> None:
self.github_pr
)
if not ckan.Staged:
self.master.append(ckan)
self.primary.append(ckan)
else:
self.staged.append(ckan)

Expand All @@ -227,14 +227,14 @@ def _process_queue(self, queue: Deque[CkanMessage]) -> None:
def sqs_delete_entries(self) -> List[DeleteMessageBatchRequestEntryTypeDef]:
return [c.delete_attrs for c in self.processed]

# Currently we intermingle Staged/Master commits
# Currently we intermingle Staged/Primary commits
# separating them out will be a little more efficient
# with our push/pull traffic.
def process_messages(self) -> None:
self._process_queue(self.master)
self._process_queue(self.primary)
if any(ckan.indexed for ckan in self.processed):
self.repo.pull_remote_branch('master')
self.repo.push_remote_branch('master')
self.repo.pull_remote_primary()
self.repo.push_remote_primary()
self._process_queue(self.staged)


Expand Down
Loading

0 comments on commit ee03890

Please sign in to comment.