Feat/use all tags (#74)

* Use all github tags for versions

* Gitlab tags

* Small refacto

---------

Co-authored-by: Antoine MICARD <antoine.micard@sii.fr>

.env-example

@@ -20,6 +20,8 @@ OTTM_CURRENT_BRANCH=devel
 OTTM_SOURCE_REPO_URL=https://github.com/dbeaver/dbeaver
 # Either "github" or "gitlab", other SCM are not yet supported
 OTTM_SOURCE_REPO_SCM=github
+# Consider all tags as versions. If False, will only take releases
+OTTM_SCM_USE_ALL_TAGS=False
 # SCM base URL - leave empty for public repo
 OTTM_SCM_BASE_URL=
 # Token to access github or gitlab
@@ -66,14 +68,14 @@ OTTM_LEGACY_MINIMUM_DAYS=365
 # The number of seconds to wait after a failed API call due to a limit of calls exceeded
 OTTM_RETRY_DELAY=3600
 
-# The url for survey api
-SURVEY_BACK_API_URL="http://localhost:8000/"
+# The url for Survey API
+SURVEY_BACK_API_URL=http://localhost:8000/
 # The name of the project for Survey API
 SURVEY_PROJECT_NAME=
 
-# The corrélation name for TOPSIS
-# Available value are pearson, spearman, kendall, weighted
-# Let it blanc for all of them
+# The correlation name for TOPSIS
+# Available values are pearson, spearman, kendall, weighted
+# Let it blank for all of them
 OTTM_CORR_METHOD=
 # The criteria name for TOPSIS
 # Available value are bugs
@@ -83,4 +85,4 @@ OTTM_CRITERIA=bugs
 OTTM_ALTERNATIVES=bug_velocity,changes,team_xp,code_churn,complexity,legacy_files
 # The criteria weigth for TOPSIS
 # You should provide the same amont as the number of criteria
-OTTM_WEIGTHS=1
+OTTM_WEIGTHS=1

configuration.py

@@ -31,9 +31,10 @@ def __init__(self):
         self.target_database = self.__get_required_value("OTTM_TARGET_DATABASE")
 
         self.source_repo_scm = self.__get_repo_scm("OTTM_SOURCE_REPO_SCM")
-        self.source_project = self.__get_required_value("OTTM_SOURCE_PROJECT")
-        self.source_repo = self.__get_required_value("OTTM_SOURCE_REPO")
-        self.current_branch = self.__get_required_value("OTTM_CURRENT_BRANCH")
+        self.use_all_tags = self.__get_bool("OTTM_SCM_USE_ALL_TAGS", False)
+        self.source_project  = self.__get_required_value("OTTM_SOURCE_PROJECT")
+        self.source_repo     = self.__get_required_value("OTTM_SOURCE_REPO")
+        self.current_branch  = self.__get_required_value("OTTM_CURRENT_BRANCH")
         self.source_repo_url = self.__get_required_value("OTTM_SOURCE_REPO_URL")
         self.source_bugs = self.__get_str_list("OTTM_SOURCE_BUGS")
 
@@ -204,3 +205,18 @@ def __get_required_value(env_var):
         if not value:
             raise ConfigurationValidationException(f"Value for {env_var} is required")
         return value
+
+    @staticmethod
+    def __get_bool(env_var, default):
+        parsed_value = os.getenv(env_var)
+        if parsed_value is None:
+            return default
+        if parsed_value.lower() == "true":
+            parsed_bool = True
+        elif parsed_value.lower() == "false":
+            parsed_bool = False
+        else:
+            raise ConfigurationValidationException(
+                f"Incorrect value : {parsed_value}, {env_var} should be a boolean"
+            )
+        return parsed_bool

connectors/git.py

@@ -225,5 +225,5 @@ def _get_issues(self, since, labels):
         raise NotImplementedError
 
     @abstractmethod
-    def _get_releases(self, all, order_by, sort):
+    def _get_git_versions(self, all, order_by, sort):
         raise NotImplementedError

connectors/github.py

@@ -1,5 +1,6 @@
 import logging
 from time import sleep, time
+from typing import List, Union
 import github
 
 from sqlalchemy import desc, update
@@ -8,6 +9,9 @@
 from models.issue import Issue
 from models.version import Version
 from github import Github
+from github.GitRelease import GitRelease
+from github.Tag import Tag
+from github.PaginatedList import PaginatedList
 import datetime
 from connectors.git import GitConnector
 from utils.timeit import timeit
@@ -37,7 +41,9 @@ def _get_issues(self, since=None, labels=None):
             sleep(self.configuration.retry_delay)
             self._get_issues(since, labels)
 
-    def _get_releases(self, all=None, order_by=None, sort=None):
+    def _get_git_versions(
+        self, all=None, order_by=None, sort=None
+    ) -> List[Union[GitRelease, Tag]]:
         if not all:
             all = None
         if not order_by:
@@ -46,10 +52,13 @@ def _get_releases(self, all=None, order_by=None, sort=None):
             sort = None
 
         try:
-            return self.remote.get_releases()
+            if self.configuration.use_all_tags:
+                return self.remote.get_tags()
+            else:
+                return self.remote.get_releases()
         except github.GithubException.RateLimitExceededException:
             sleep(self.configuration.retry_delay)
-            self._get_releases(all, order_by, sort)
+            self._get_git_versions(all, order_by, sort)
 
     @timeit
     def create_issues(self):
@@ -129,7 +138,7 @@ def create_versions(self):
         Create versions into the database from GitHub tags
         """
         logging.info("GitHubConnector: create_versions")
-        releases = self._get_releases()
+        git_versions = self._get_git_versions()
         self._clean_project_existing_versions()
 
         versions = []
@@ -139,25 +148,34 @@ def create_versions(self):
         forks = list(self.remote.get_forks())
         subscribers = list(self.remote.get_subscribers())
 
-        for release in releases.reversed:
+        for v in git_versions.reversed:
+            if type(v) is GitRelease:
+                v_name = v.title
+                v_tag = v.tag_name
+                v_end_date = v.published_at
+            elif type(v) is Tag:
+                v_name = v.name
+                v_tag = v.name
+                v_end_date = v.commit.commit.committer.date
+
             # Set UTC Timezone for previous release and release published_at when they are None
             if previous_release_published_at.tzinfo is None:
                 previous_release_published_at = (
                     previous_release_published_at.astimezone(datetime.timezone.utc)
                 )
 
-            if release.published_at.tzinfo is None:
-                release_published_at_timezone = release.published_at.astimezone(
+            if v_end_date.tzinfo is None:
+                release_published_at_timezone = v_end_date.astimezone(
                     datetime.timezone.utc
                 )
 
             versions.append(
                 Version(
                     project_id=self.project_id,
-                    name=release.title,
-                    tag=release.tag_name,
+                    name=v_name,
+                    tag=v_tag,
                     start_date=previous_release_published_at,
-                    end_date=release.published_at,
+                    end_date=v_end_date,
                     stars=len(
                         list(
                             filter(
@@ -204,7 +222,7 @@ def create_versions(self):
                     ),
                 )
             )
-            previous_release_published_at = release.published_at
+            previous_release_published_at = v_end_date
 
         # Put current branch at the end of the list
         # Set UTC Timezone for previous release published_at when it's None

connectors/gitlab.py

@@ -1,5 +1,6 @@
 import logging
 from time import sleep
+from typing import List, Union
 import gitlab
 
 from sqlalchemy import desc, update
@@ -10,6 +11,8 @@
 from utils.timeit import timeit
 from connectors.git import GitConnector
 from gitlab import Gitlab
+from gitlab.v4.objects.tags import ProjectTag
+from gitlab.v4.objects.releases import ProjectRelease
 from datetime import datetime, timedelta
 
 
@@ -59,21 +62,24 @@ def _get_issues(self, since=None, labels=None):
         except gitlab.GitlabJobRetryError:
             sleep(self.configuration.retry_delay)
             self._get_issues(since, labels)
-
-    def _get_releases(self, all, order_by, sort):
+    
+    def _get_git_versions(self, all=None, order_by=None, sort=None) -> List[Union[ProjectRelease, ProjectTag]]:
         if not all:
             all = None
         if not order_by:
             order_by = None
         if not sort:
             sort = None
-
+        
         try:
-            return self.remote.releases.list(all=all, order_by=order_by, sort=sort)
+            if self.configuration.use_all_tags:
+                return self.remote.tags.list(all=all, order_by=order_by, sort=sort)
+            else:
+                return self.remote.releases.list(all=all, order_by=order_by, sort=sort)
         except gitlab.GitlabJobRetryError:
             sleep(self.configuration.retry_delay)
-            self._get_releases(all, order_by, sort)
-
+            self._get_git_versions(all, order_by, sort)
+        
     @timeit
     def create_issues(self):
         """
@@ -150,25 +156,41 @@ def create_versions(self):
         """
         Create versions into the database from GitLab releases
         """
-        logging.info("GitLabConnector: create_versions")
-        releases = self._get_releases(all=True, order_by="released_at", sort="asc")
+        logging.info('GitLabConnector: create_versions')
+        if self.configuration.use_all_tags:
+            git_versions = self._get_git_versions(all=True, order_by="updated", sort="asc")
+        else:
+            git_versions = self._get_git_versions(all=True, order_by="released_at", sort="asc")
         self._clean_project_existing_versions()
 
         versions = []
         previous_release_published_at = self._get_first_commit_date()
 
-        for release in releases:
-            release_published_at = date_iso_8601_to_datetime(release.released_at)
-            versions.append(
-                Version(
-                    project_id=self.project_id,
-                    name=release.name,
-                    tag=release.tag_name,
-                    start_date=previous_release_published_at,
-                    end_date=release_published_at,
+        for v in git_versions:
+            if type(v) is ProjectRelease:
+                release_published_at = date_iso_8601_to_datetime(v.released_at)
+                versions.append(
+                    Version(
+                        project_id=self.project_id,
+                        name=v.name,
+                        tag=v.tag_name,
+                        start_date=previous_release_published_at,
+                        end_date=release_published_at,
+                    )
                 )
-            )
-            previous_release_published_at = release_published_at
+                previous_release_published_at = release_published_at
+            elif type(v) is ProjectTag:
+                release_published_at = date_iso_8601_to_datetime(v.commit["committed_date"])
+                versions.append(
+                    Version(
+                        project_id=self.project_id,
+                        name=v.name,
+                        tag=v.name,
+                        start_date=previous_release_published_at,
+                        end_date=release_published_at,
+                    )
+                )
+                previous_release_published_at = release_published_at
 
         # Put current branch at the end of the list
         versions.append(