Skip to content

Commit

Permalink
Fix getting issues from GitHub/GitLab (#71)
Browse files Browse the repository at this point in the history 
* fix(issues): Getting issues from GitHub and GitLab by their labels

* Delete get_bugs_issues() method
  • Loading branch information
@ianlcz
ianlcz authored Aug 18, 2023
1 parent 7ee77dd commit a53c557
Show file tree
Hide file tree
Showing 3 changed files with 95 additions and 64 deletions.
74 changes: 42 additions & 32 deletions configuration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,53 +8,53 @@

AVAILABLE_SCM = ["github", "gitlab"]


class Configuration:

next_version_name = "Next Release"

def __init__(self):

self.log_level = self.__get_log_level("OTTM_LOG_LEVEL")

self.code_maat_path = self.__get_external_tool("OTTM_CODE_MAAT_PATH")
self.code_ck_path = self.__get_external_tool("OTTM_CODE_CK_PATH")
self.code_maat_path = self.__get_external_tool("OTTM_CODE_MAAT_PATH")
self.code_ck_path = self.__get_external_tool("OTTM_CODE_CK_PATH")
self.code_jpeek_path = self.__get_external_tool("OTTM_CODE_JPEEK_PATH")
self.code_pdepend_path = self.__get_external_tool("OTTM_CODE_PDEPEND_PATH")

self.language = os.getenv("OTTM_LANGUAGE", "")

self.scm_path = self.__get_executable("OTTM_SCM_PATH")
self.scm_path = self.__get_executable("OTTM_SCM_PATH")
if self.language.lower() == "java":
self.java_path = self.__get_executable("OTTM_JAVA_PATH")
elif self.language.lower() == "php":
self.php_path = self.__get_executable("OTTM_PHP_PATH")

self.target_database = self.__get_required_value("OTTM_TARGET_DATABASE")

self.source_repo_scm = self.__get_repo_scm("OTTM_SOURCE_REPO_SCM")
self.source_project = self.__get_required_value("OTTM_SOURCE_PROJECT")
self.source_repo = self.__get_required_value("OTTM_SOURCE_REPO")
self.current_branch = self.__get_required_value("OTTM_CURRENT_BRANCH")
self.source_project = self.__get_required_value("OTTM_SOURCE_PROJECT")
self.source_repo = self.__get_required_value("OTTM_SOURCE_REPO")
self.current_branch = self.__get_required_value("OTTM_CURRENT_BRANCH")
self.source_repo_url = self.__get_required_value("OTTM_SOURCE_REPO_URL")
self.source_bugs = self.__get_str_list("OTTM_SOURCE_BUGS")
self.source_bugs = self.__get_str_list("OTTM_SOURCE_BUGS")

self.survey_back_api_url = os.getenv("SURVEY_BACK_API_URL","")
self.survey_back_api_url = os.getenv("SURVEY_BACK_API_URL", "")
self.survey_project_name = self.__get_str_list("SURVEY_PROJECT_NAME")
self.scm_base_url = os.getenv("OTTM_SCM_BASE_URL", "")
self.scm_token = os.getenv("OTTM_SCM_TOKEN", "")

self.jira_base_url = os.getenv("OTTM_JIRA_BASE_URL", "")
self.jira_project = os.getenv("OTTM_JIRA_PROJECT", "")
self.jira_email = os.getenv("OTTM_JIRA_EMAIL", "")
self.jira_token = os.getenv("OTTM_JIRA_TOKEN", "")
self.scm_base_url = os.getenv("OTTM_SCM_BASE_URL", "")
self.scm_token = os.getenv("OTTM_SCM_TOKEN", "")
self.scm_issues_labels = os.getenv("OTTM_SCM_ISSUES_LABELS", "")

self.jira_base_url = os.getenv("OTTM_JIRA_BASE_URL", "")
self.jira_project = os.getenv("OTTM_JIRA_PROJECT", "")
self.jira_email = os.getenv("OTTM_JIRA_EMAIL", "")
self.jira_token = os.getenv("OTTM_JIRA_TOKEN", "")
self.jira_issue_type = self.__get_str_list("OTTM_JIRA_ISSUE_TYPE")

self.glpi_categories = self.__get_str_list("OTTM_GLPI_CATEGORIES")
self.glpi_base_url = os.getenv("OTTM_GLPI_BASE_URL", "")
self.glpi_app_token = os.getenv("OTTM_GLPI_APP_TOKEN", "")
self.glpi_base_url = os.getenv("OTTM_GLPI_BASE_URL", "")
self.glpi_app_token = os.getenv("OTTM_GLPI_APP_TOKEN", "")
self.glpi_user_token = os.getenv("OTTM_GLPI_USER_TOKEN", "")
self.glpi_username = os.getenv("OTTM_GLPI_USERNAME", "")
self.glpi_password = os.getenv("OTTM_GLPI_PASSWORD", "")
self.glpi_username = os.getenv("OTTM_GLPI_USERNAME", "")
self.glpi_password = os.getenv("OTTM_GLPI_PASSWORD", "")

self.issue_tags = self.__get_str_list("OTTM_ISSUE_TAGS")
self.exclude_issuers = self.__get_str_list("OTTM_EXCLUDE_ISSUERS")
Expand All @@ -71,7 +71,7 @@ def __init__(self):
self.insignificant_commits_message = self.__get_str_list("OTTM_COMMIT_BAD_MSG")

self.retry_delay = self.__get_retry_delay("OTTM_RETRY_DELAY")

self.legacy_percent = self.__get_float("OTTM_LEGACY_PERCENT", 20)

self.legacy_minimum_days = self.__get_int("OTTM_LEGACY_MINIMUM_DAYS", 365)
Expand All @@ -97,27 +97,37 @@ def __get_log_level(env_var):
elif required_level == "NOTSET":
log_level = logging.NOTSET
else:
raise ConfigurationValidationException(f"Invalid value for log level: {required_level}")
raise ConfigurationValidationException(
f"Invalid value for log level: {required_level}"
)

return log_level

@staticmethod
def __get_external_tool(env_var):
if env_var not in os.environ:
raise ConfigurationValidationException(f"No external tool specified for ${env_var}")
raise ConfigurationValidationException(
f"No external tool specified for ${env_var}"
)
file_path = os.environ[env_var]
if not os.path.exists(file_path):
raise ConfigurationValidationException(f"The following external tool was not found: {file_path}")
raise ConfigurationValidationException(
f"The following external tool was not found: {file_path}"
)
return file_path

@staticmethod
def __get_executable(env_var):
if env_var not in os.environ:
raise ConfigurationValidationException(f"No executable specified for ${env_var}")
raise ConfigurationValidationException(
f"No executable specified for ${env_var}"
)
executable = os.environ[env_var]
executable_found = shutil.which(executable)
if not executable_found:
raise ConfigurationValidationException(f"The following executable was not found: {executable}")
raise ConfigurationValidationException(
f"The following executable was not found: {executable}"
)
return executable

@staticmethod
Expand All @@ -127,8 +137,8 @@ def __get_repo_scm(env_var) -> str:
raise ConfigurationValidationException(f"No source code manager specified")
if repo_scm not in AVAILABLE_SCM:
raise ConfigurationValidationException(
f"The following source code manager is not handled by OTTM : {repo_scm}." +\
f" Availables SCM are : {AVAILABLE_SCM}"
f"The following source code manager is not handled by OTTM : {repo_scm}."
+ f" Availables SCM are : {AVAILABLE_SCM}"
)
return repo_scm

Expand Down Expand Up @@ -161,7 +171,7 @@ def __get_float(env_var, default):
)
return parsed_float

@staticmethod
@staticmethod
def __get_int(env_var, default):
parsed_value = os.getenv(env_var)
if parsed_value is None:
Expand Down
6 changes: 3 additions & 3 deletions connectors/github.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def _get_issues(self, since=None, labels=None):
labels = github.GithubObject.NotSet

try:
return self.remote.get_issues(state="all", since=since, labels=labels)
return self.remote.get_issues(state="all", labels=labels)
except github.GithubException.RateLimitExceededException:
sleep(self.configuration.retry_delay)
self._get_issues(since, labels)
Expand Down Expand Up @@ -68,7 +68,7 @@ def create_issues(self):
)
if last_issue is not None:
# Update existing database by fetching new issues
if not self.configuration.issue_tags:
if len(self.configuration.issue_tags) == 0:
git_issues = self._get_issues(
since=last_issue.updated_at + datetime.timedelta(seconds=1)
)
Expand All @@ -79,7 +79,7 @@ def create_issues(self):
) # e.g. Filter by labels=['bug']
else:
# Create a database with all issues
if not self.configuration.issue_tags:
if len(self.configuration.issue_tags) == 0:
git_issues = self._get_issues()
else:
git_issues = self._get_issues(
Expand Down
79 changes: 50 additions & 29 deletions connectors/gitlab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from gitlab import Gitlab
from datetime import datetime, timedelta


class GitLabConnector(GitConnector):
"""
Connector to GitLab
Expand All @@ -20,18 +21,25 @@ class GitLabConnector(GitConnector):
-----------
- base_url URL to GitLab, empty if gitlab.com
"""
def __init__(self, project_id, directory, base_url, token, repo, current, session, config):
GitConnector.__init__(self, project_id, directory, token, repo, current, session, config)

def __init__(
self, project_id, directory, base_url, token, repo, current, session, config
):
GitConnector.__init__(
self, project_id, directory, token, repo, current, session, config
)
if not base_url and not self.token:
logging.info("anonymous read-only access for public resources (GitLab.com)")
self.api = Gitlab()
if base_url and self.token:
logging.info("private token or personal token authentication (self-hosted GitLab instance)")
logging.info(
"private token or personal token authentication (self-hosted GitLab instance)"
)
self.api = Gitlab(url=base_url, private_token=self.token)
if not base_url and self.token:
logging.info("private token or personal token authentication (GitLab.com)")
self.api = Gitlab(private_token=self.token)

# Check the authentification. Doesn't work for public read only access
if base_url or self.token:
self.api.auth()
Expand All @@ -45,69 +53,82 @@ def _get_issues(self, since=None, labels=None):
labels = None

try:
return self.remote.issues.list(state="all", since=since, with_labels_details=labels, get_all=True)
return self.remote.issues.list(
state="all", since=since, with_labels_details=labels, get_all=True
)
except gitlab.GitlabJobRetryError:
sleep(self.configuration.retry_delay)
self._get_issues(since, labels)

def _get_releases(self, all, order_by, sort):
if not all:
all = None
if not order_by:
order_by = None
if not sort:
sort = None

try:
return self.remote.releases.list(all=all, order_by=order_by,sort=sort)
return self.remote.releases.list(all=all, order_by=order_by, sort=sort)
except gitlab.GitlabJobRetryError:
sleep(self.configuration.retry_delay)
self._get_releases(all, order_by, sort)

@timeit
def create_issues(self):
"""
Create issues into the database from GitLab Issues
"""
logging.info('GitLabConnector: create_issues')
logging.info("GitLabConnector: create_issues")

# Check if a database already exist
last_issue = self.session.query(Issue) \
.filter(Issue.project_id == self.project_id) \
.filter(Issue.source == 'git') \
.order_by(desc(Issue.updated_at)).first()
last_issue = (
self.session.query(Issue)
.filter(Issue.project_id == self.project_id)
.filter(Issue.source == "git")
.order_by(desc(Issue.updated_at))
.first()
)
if last_issue is not None:
# Update existing database by fetching new issues
if not self.configuration.issue_tags:
git_issues = self._get_issues(since=last_issue.updated_at + timedelta(seconds=1), labels=None)
if len(self.configuration.issue_tags) == 0:
git_issues = self._get_issues(
since=last_issue.updated_at + timedelta(seconds=1), labels=None
)
else:
git_issues = self._get_issues(since=last_issue.updated_at + timedelta(seconds=1),
labels=self.configuration.issue_tags) # e.g. Filter by labels=['bug']
git_issues = self._get_issues(
since=last_issue.updated_at + timedelta(seconds=1),
labels=self.configuration.issue_tags,
) # e.g. Filter by labels=['bug']
else:
# Create a database with all issues
if not self.configuration.issue_tags:
if len(self.configuration.issue_tags) == 0:
git_issues = self._get_issues(since=None, labels=None)
else:
git_issues = self._get_issues(labels=self.configuration.issue_tags) # e.g. Filter by labels=['bug']

git_issues = self._get_issues(
labels=self.configuration.issue_tags
) # e.g. Filter by labels=['bug']

# versions = self.session.query(Version).all
logging.info('Syncing ' + str(len(git_issues)) + ' issue(s) from GitLab')
logging.info("Syncing " + str(len(git_issues)) + " issue(s) from GitLab")

new_bugs = []
# for version in versions:
for issue in git_issues:
# Check if the issue is linked to a selected version (included or not +IN?.§ .?NBVCXd)
# if version.end_date > issue.created_at > version.start_date:
if issue.author['username'] not in self.configuration.exclude_issuers:

if issue.author["username"] not in self.configuration.exclude_issuers:
updated_issue_date = date_iso_8601_to_datetime(issue.updated_at)
existing_issue_id = self._get_existing_issue_id(issue.iid)

if existing_issue_id:
logging.info("Issue %s already exists, updating it", existing_issue_id)
logging.info(
"Issue %s already exists, updating it", existing_issue_id
)
self.session.execute(
update(Issue).where(Issue.issue_id == existing_issue_id) \
.values(title=issue.title, updated_at=updated_issue_date)
update(Issue)
.where(Issue.issue_id == existing_issue_id)
.values(title=issue.title, updated_at=updated_issue_date)
)
else:
new_bugs.append(
Expand All @@ -123,13 +144,13 @@ def create_issues(self):

self.session.add_all(new_bugs)
self.session.commit()

@timeit
def create_versions(self):
"""
Create versions into the database from GitLab releases
"""
logging.info('GitLabConnector: create_versions')
logging.info("GitLabConnector: create_versions")
releases = self._get_releases(all=True, order_by="released_at", sort="asc")
self._clean_project_existing_versions()

Expand Down

0 comments on commit a53c557

Please sign in to comment.