diff --git a/test/repos/tasks/test_sync_repos.py b/test/repos/tasks/test_sync_repos.py index 3a4a7f5c..b8739bd9 100644 --- a/test/repos/tasks/test_sync_repos.py +++ b/test/repos/tasks/test_sync_repos.py @@ -16,6 +16,7 @@ def __init__(self, id=None, name=None, external_url=None, slug=None): self.name = self.fake.word() if name is None else name self.slug = self.fake.word() if slug is None else slug self.external_url = self.fake.url() if external_url is None else external_url + self.state = "available" class FakeGitProject: diff --git a/zoo/repos/tasks.py b/zoo/repos/tasks.py index 7c8c25a4..c9475429 100644 --- a/zoo/repos/tasks.py +++ b/zoo/repos/tasks.py @@ -41,6 +41,8 @@ @shared_task def sync_repos(): + + i = 0 for project in itertools.chain( get_github_repositories(), get_gitlab_repositories() ): @@ -48,7 +50,8 @@ def sync_repos(): continue if settings.SYNC_REPOS_SKIP_PERSONAL and project["is_personal"]: continue - + i += 1 + log.info("sync_repos.fetch", repo_number=i, project=project) try: repo = Repository.objects.get( remote_id=project["id"], provider=project["provider"] @@ -71,8 +74,11 @@ def sync_repos(): repo.save() if project["provider"] == "gitlab": + log.info("sync_repos.calling.sync_enviroments_from_gitlab") sync_enviroments_from_gitlab(repo) + log.info("sync_repos.total", repo_number=i) + @shared_task def schedule_pulls(): @@ -219,16 +225,22 @@ def sync_enviroments_from_gitlab(repo: Repository): gl_envs = get_project_enviroments(repo.remote_id) if not gl_envs: + log.info("sync_enviroments_from_gitlab.no_envs") return envs = [] + i = 0 + log.info("sync_enviroments_from_gitlab.start") for gl_env in gl_envs: + if gl_env.state != "available": + continue env, _ = RepositoryEnvironment.objects.get_or_create( repository_id=repo.id, name=gl_env.name ) env.external_url = gl_env.external_url env.save() envs.append(env) + i += 1 RepositoryEnvironment.objects.filter(repository_id=repo.id).exclude( id__in=[env.id for env in envs] @@ -241,9 +253,12 @@ def sync_enviroments_from_gitlab(repo: Repository): service_id=service.id, type=EnviromentType.GITLAB.value ).exclude(name__in=[env.name for env in envs]).delete() for env in envs: - Environment.objects.update_or_create( + service_env, _ = Environment.objects.get_or_create( service_id=service.id, name=env.name, - dashboard_url=env.external_url, type=EnviromentType.GITLAB.value, ) + service_env.dashboard_url = env.external_url + service_env.save() + + log.info("sync_enviroments_from_gitlab.total.synced", envs_synced=i) diff --git a/zoo/repos/templates/repos/fields/repo_input.html b/zoo/repos/templates/repos/fields/repo_input.html index 42108849..b9daa47f 100644 --- a/zoo/repos/templates/repos/fields/repo_input.html +++ b/zoo/repos/templates/repos/fields/repo_input.html @@ -23,8 +23,6 @@ initialValue: '{{ widget.value|default:"" }}' }; const gitlabEnvsInfo = { - envTypeGitlab: '{{ env_type_gitlab }}', - imageUrl: '{% static 'img/gitlab-icon.svg' %}', envsUrl: '{% url "get_gitlab_envs" %}' } diff --git a/zoo/repos/views.py b/zoo/repos/views.py index d5c7f0a8..70de8c0c 100644 --- a/zoo/repos/views.py +++ b/zoo/repos/views.py @@ -44,7 +44,8 @@ def get_gitlab_envs(request): return JsonResponse( [ {"name": gl_env.name, "dashboardUrl": gl_env.external_url} - for gl_env in repo.repository_environments.all() + for index, gl_env in enumerate(repo.repository_environments.all()) + if index < 10 ], safe=False, ) diff --git a/zoo/services/assets/js/gitlab_envs.js b/zoo/services/assets/js/gitlab_envs.js index 2ae497ae..ef1f322e 100644 --- a/zoo/services/assets/js/gitlab_envs.js +++ b/zoo/services/assets/js/gitlab_envs.js @@ -1,76 +1,31 @@ const gitlabEnvs = { - _resetGilabEnvs: (envTypeGitlab) => { - const gitlab_envs = $("input[value='" + envTypeGitlab + "']").closest(".ui.segment"); - if (gitlab_envs.length > 0) { - $.each(gitlab_envs, function(index, element){ - const uiSegment = $(element) - - // set values to default - uiSegment.find(".field.name input").val(""); - uiSegment.find('input[name$="type"]').val(""); - uiSegment.find('input[name$="dashboard_url"]').val(""); - uiSegment.find('input[name$="logs_url"]').val(""); - uiSegment.find('input[name$="open_api_url"]').val(""); - - // remove gitlab icon - uiSegment.find(".gitlab-img").hide() - - uiSegment.hide(); - uiSegment.find(".field.name input").prop("required", false); - - //if all segments are hidden, - const hidden = $(".ui.segment.environment:hidden"); - if (hidden.length == 5) { - let firstSegment = hidden.eq(0); - firstSegment.show() - } - - if (hidden.length > 0) { - $("button.add-environment").prop("disabled", false); - } - - }) - } - }, - _getFirstAvailableSegment: () => { - const all_envs = $(".ui.segment.environment"); - if (all_envs.length == 0) { return false } - - for (let i = 0; i < all_envs.length; i++) { - let firstSegment = all_envs.eq(i) - if (firstSegment.find(".field.name input").val() == "") { - return firstSegment - } - } - - return false; - }, - _fillData: (envTypeGitlab, firstSegment, response) => { - firstSegment.find(":not(.no-reset) input").val(""); - firstSegment.find('input[name$="type"]').val(envTypeGitlab); - firstSegment.find(".field.name input").val(response["name"]).prop('readonly', true); - firstSegment.find('input[name$="dashboard_url"]').val(response["dashboardUrl"]).prop('readonly', true); - firstSegment.find(".gitlab-img").show() - firstSegment.show(); + _fillData: (response) => { + const envName = response['name'] || "" + const envDashboardUrl = response['dashboardUrl'] || "" + + let $env = $('#env_template').clone() + $env.find("input[name='gitlab-name']").val(envName) + $env.find("input[name='gitlab-dashboard-url']").val(envDashboardUrl) + $env.appendTo("#gitlab_envs" ) + $env.show() }, load: (gitlabEnvsInfo) => { if (Number.isNaN(gitlabEnvsInfo.repoId)) { return } //delete previous gitlab envs - gitlabEnvs._resetGilabEnvs(gitlabEnvsInfo.envTypeGitlab) - + $("#gitlab_envs").empty() //fetch gitlab envs $.get(gitlabEnvsInfo.envsUrl + "?project_id=" + gitlabEnvsInfo.repoId, function (response) { - if (response.length == 0) { return console.log('Fetching Gitlab environments completed! No envs.') } + let customEnvsNameInput = $(".ui.segment.environment:visible").find(".field.name input") + if (response.length == 0) { + customEnvsNameInput.prop("required", true); + return console.log('Fetching Gitlab environments completed! No envs.') + } + customEnvsNameInput.prop("required", false); $.each(response, function(index, element) { - let firstSegment = gitlabEnvs._getFirstAvailableSegment(); - if (!firstSegment) { return;} - - gitlabEnvs._fillData(gitlabEnvsInfo.envTypeGitlab, firstSegment, element) - if ($(".ui.segment.environment:hidden").length == 0) { - $("button.add-environment").prop("disabled", true); - } + if (index > 10) { return;} + gitlabEnvs._fillData(element) }) console.log('Fetching Gitlab environments completed!') }).fail(function() { diff --git a/zoo/services/assets/js/service_form.js b/zoo/services/assets/js/service_form.js index 5692cdbf..8fc2353f 100644 --- a/zoo/services/assets/js/service_form.js +++ b/zoo/services/assets/js/service_form.js @@ -94,3 +94,9 @@ $('.icon-hint .label').popup({ variation: 'inverted', position: 'left center' }); + +$(".ui.green.button").on('submit', function(e) { + e.preventDefault(); + $("#gitlab_envs input").prop('disabled', true) + $(this).submit(); +}); diff --git a/zoo/services/forms.py b/zoo/services/forms.py index bcbd2a7d..4fc572dc 100644 --- a/zoo/services/forms.py +++ b/zoo/services/forms.py @@ -16,6 +16,7 @@ from ..repos.forms import RepoInput from ..repos.github import get_namespaces as get_github_namespaces from ..repos.gitlab import get_namespaces as get_gitlab_namespaces +from ..repos.models import Provider from . import models @@ -58,7 +59,6 @@ class Meta: "logs_url", "service_urls", "open_api_url", - "type", ] labels = { "dashboard_url": "Dashboard URL", @@ -178,7 +178,25 @@ def save(self, commit=True): repository.save() except (KeyError, AttributeError): pass - return super().save() + + service = super().save() + self.create_gitlab_envs(service) + return service + + def create_gitlab_envs(self, service): + if not service.repository: + return + + if service.repository.provider == Provider.GITHUB: + return + + for env in service.repository.repository_environments.all(): + models.Environment.objects.update_or_create( + type=models.EnviromentType.GITLAB, + name=env.name, + dashboard_url=env.external_url, + service_id=service.id, + ) ServiceEnvironmentsFormSet = forms.inlineformset_factory( diff --git a/zoo/services/migrations/0027_auto_20210509_1900.py b/zoo/services/migrations/0027_auto_20210509_1900.py new file mode 100644 index 00000000..8d5a24a2 --- /dev/null +++ b/zoo/services/migrations/0027_auto_20210509_1900.py @@ -0,0 +1,33 @@ +# Generated by Django 2.2.19 on 2021-05-09 19:00 + +from django.db import migrations, models + +import zoo.services.constants + + +class Migration(migrations.Migration): + + dependencies = [ + ("services", "0026_link"), + ] + + operations = [ + migrations.AddField( + model_name="environment", + name="type", + field=models.CharField( + choices=[("gitlab", "gitlab"), ("zoo", "zoo")], + default=zoo.services.constants.EnviromentType("zoo"), + max_length=100, + ), + ), + migrations.AlterField( + model_name="environment", + name="name", + field=models.CharField(max_length=200), + ), + migrations.AlterUniqueTogether( + name="environment", + unique_together={("service", "name", "type")}, + ), + ] diff --git a/zoo/services/migrations/0027_environment_type.py b/zoo/services/migrations/0027_environment_type.py deleted file mode 100644 index 1485f483..00000000 --- a/zoo/services/migrations/0027_environment_type.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 2.2.19 on 2021-04-23 13:58 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ("services", "0026_link"), - ] - - operations = [ - migrations.AddField( - model_name="environment", - name="type", - field=models.CharField( - choices=[("gitlab", "gitlab"), ("zoo", "zoo")], - max_length=100, - null=True, - ), - ), - ] diff --git a/zoo/services/models.py b/zoo/services/models.py index 8ff73e2e..f2803231 100644 --- a/zoo/services/models.py +++ b/zoo/services/models.py @@ -174,7 +174,7 @@ def __repr__(self): class Environment(models.Model): class Meta: - unique_together = ("service", "name") + unique_together = ("service", "name", "type") ordering = ["name"] service = models.ForeignKey( @@ -183,7 +183,7 @@ class Meta: related_name="environments", related_query_name="environment", ) - name = models.CharField(max_length=100) + name = models.CharField(max_length=200) service_urls = pg_fields.ArrayField( models.URLField(max_length=500), null=True, default=list ) @@ -193,8 +193,8 @@ class Meta: open_api_url = models.URLField(max_length=500, null=True, blank=True) type = models.CharField( choices=((item.value, item.value) for item in EnviromentType), - null=True, max_length=100, + default=EnviromentType.ZOO, ) diff --git a/zoo/services/templates/services/service_detail.html b/zoo/services/templates/services/service_detail.html index c1b442cc..30003958 100644 --- a/zoo/services/templates/services/service_detail.html +++ b/zoo/services/templates/services/service_detail.html @@ -57,12 +57,12 @@ docs - {% if object.environments_dict|length >= 1 %} + {% if environments_dict|length >= 1 %}
env:{{ environment.name }}

Environments

- {% for name in object.environments_dict %} + {% for name in environments_dict %} {{ name }} diff --git a/zoo/services/templates/services/service_form.html b/zoo/services/templates/services/service_form.html index 7e66a1b4..28916b31 100644 --- a/zoo/services/templates/services/service_form.html +++ b/zoo/services/templates/services/service_form.html @@ -30,81 +30,128 @@

Environments

-
- -
+
-
- {{ envs_formset.management_form }} - {% for form in envs_formset %} -
1 %}style="display: none"{% endif %}> - - {% if forloop.counter > 1 and form.type != env_type_gitlab %} -
- + {{ envs_formset.management_form }} + {% for form in envs_formset %} +
1 %}style="display: none"{% endif %}> + {% if forloop.counter > 1 %} +
+ +
+ {% endif %} + {% if form.errors %} +
+
Please correct the following errors
+
    + {% for field, messages in form.errors.items %} +
  • {{ field }} - {{ messages|join:" " }}
    • + {% endfor %} +
+
+ {% endif %} +
+ {% for field in form %} + {% with field.field.widget.input_type as type %} + {% if type == "hidden" %} + {{ field }} + {% endif %} + {% endwith %} + {% endfor %}
- {% endif %} - {% if form.errors %} -
-
Please correct the following errors
-
    - {% for field, messages in form.errors.items %} -
  • {{ field }} - {{ messages|join:" " }}
    • - {% endfor %} -
+ {{ form.DELETE.as_hidden }} +
+
+ + {{ form.name }} +
+
+ + {{ form.dashboard_url }} +
- {% endif %} -
- {% for field in form %} - {% with field.field.widget.input_type as type %} - {% if type == "hidden" %} - {{ field }} - {% endif %} - {% endwith %} +
+ + {{ form.logs_url }} +
+
+ + {{ form.service_urls }} +
+
+ + {{ form.open_api_url }} +
+
+ {% endfor %} +
+ {% if envs_gitlab %} + {% for env in envs_gitlab %} +
+
+ gitlab +
+ +
+
+ + +
+
+ + +
+
+
+ + +
+
+ + +
+
+ + +
+
{% endfor %} + {% endif %} +
+ - {% endfor %} - +

Links

@@ -171,8 +218,5 @@

Links

{% block scripts %} {{ block.super }} - {% endblock %} diff --git a/zoo/services/views.py b/zoo/services/views.py index 1a8b213f..c76fa09d 100644 --- a/zoo/services/views.py +++ b/zoo/services/views.py @@ -16,7 +16,7 @@ from ..checklists.steps import STEPS from ..repos.utils import openapi_definition from . import forms, models -from .models import EnviromentType, Service +from .models import EnviromentType, Environment, Service log = structlog.get_logger() @@ -81,8 +81,6 @@ def get_context_data(self, **kwargs): else: data["envs_formset"] = forms.ServiceEnvironmentsFormSet() data["links_formset"] = forms.ServiceLinksFormSet() - data["env_type_gitlab"] = EnviromentType.GITLAB.value - data["env_type_zoo"] = EnviromentType.ZOO.value return data @@ -166,6 +164,7 @@ def get_context_data(self, **kwargs): context["environment"] = self.object.get_environment( self.request.GET.get("environment") ) + context["environments_dict"] = list(self.object.environments_dict.keys())[:10] return context @@ -223,12 +222,12 @@ def get_context_data(self, **kwargs): self.request.POST, instance=self.object ) else: + data["envs_gitlab"] = Environment.objects.filter(type=EnviromentType.GITLAB) data["envs_formset"] = forms.ServiceEnvironmentsFormSet( - instance=self.object + instance=self.object, + queryset=Environment.objects.filter(type=EnviromentType.ZOO), ) data["links_formset"] = forms.ServiceLinksFormSet(instance=self.object) - data["env_type_gitlab"] = EnviromentType.GITLAB.value - data["env_type_zoo"] = EnviromentType.ZOO.value return data