fix unittests

examples/example_app/example_app/jobs.py

@@ -114,11 +114,6 @@ class ExampleLoggingJob(Job):
     class Meta:
         name = "Example logging job."
         description = "I log stuff to demonstrate how UI logging works."
-        task_queues = [
-            settings.CELERY_TASK_DEFAULT_QUEUE,
-            "priority",
-            "bulk",
-        ]
 
     def run(self, interval):
         self.logger.debug("Running for %s seconds.", interval)

nautobot/extras/jobs.py

@@ -49,7 +49,7 @@
     ObjectChange,
 )
 from nautobot.extras.registry import registry
-from nautobot.extras.utils import change_logged_models_queryset
+from nautobot.extras.utils import change_logged_models_queryset, task_queues_as_choices # noqa: F401
 from nautobot.ipam.formfields import IPAddressFormField, IPNetworkFormField
 from nautobot.ipam.validators import (
     MaxPrefixLengthValidator,

nautobot/extras/models/jobs.py

@@ -299,6 +299,8 @@ def task_queues(self):
     def task_queues(self, value):
         # value is going to be a comma separated list of queue names
         # assume all queue types are celery for now
+        if isinstance(value, str):
+            value = value.split(",")
         for queue in value:
             job_queue, _ = JobQueue.objects.get_or_create(name=queue, queue_type=JobQueueTypeChoices.TYPE_CELERY)
             JobQueueAssignment.objects.get_or_create(job_queue=job_queue, job=self)
@@ -1133,8 +1135,9 @@ def queue(self):
 
     @queue.setter
     def queue(self, value):
-        job_queue = JobQueue.objects.get_or_create(name=value, type=JobQueueTypeChoices.TYPE_CELERY)
-        self.job_queue = job_queue
+        if value:
+            job_queue = JobQueue.objects.get_or_create(name=value, queue_type=JobQueueTypeChoices.TYPE_CELERY)
+            self.job_queue = job_queue
 
     @staticmethod
     def earliest_possible_time():

nautobot/extras/templatetags/job_buttons.py

@@ -108,7 +108,8 @@ def _render_job_button_for_obj(job_button, obj, context, content_type):
     # Disable buttons if the user doesn't have permission to run the underlying Job.
     has_run_perm = Job.objects.check_perms(context["user"], instance=job_button.job, action="run")
     try:
-        _job_queue = list(job_button.job.job_queues.all().values_list("name", flat=True))
+        job_queues = list(job_button.job.job_queues.all().values_list("name", flat=True))
+        _job_queue = job_queues[0]
     except IndexError:
         _job_queue = settings.CELERY_TASK_DEFAULT_QUEUE
     hidden_inputs = format_html(

nautobot/extras/test_jobs/api_test_job.py

@@ -11,7 +11,6 @@ class APITestJob(Job):
     class Meta:
         name = "Job for API Tests"
         has_sensitive_variables = False
-        task_queues = [settings.CELERY_TASK_DEFAULT_QUEUE, "nonexistent"]
 
     var1 = StringVar()
     var2 = IntegerVar(required=True)  # explicitly stated, though required=True is the default in any case

nautobot/extras/tests/test_api.py

@@ -1360,8 +1360,7 @@ class JobTest(
         "time_limit": 650,
         "has_sensitive_variables": False,
         "has_sensitive_variables_override": True,
-        "task_queues": ["default", "priority"],
-        "task_queues_override": True,
+        "job_queues": [],
     }
     bulk_update_data = {
         "enabled": True,

nautobot/extras/tests/test_jobs.py

@@ -71,7 +71,7 @@ def test_field_order(self):
         name = "TestFieldOrder"
         job_class = get_job(f"{module}.{name}")
         form = job_class().as_form()
-        self.assertSequenceEqual(list(form.fields.keys()), ["var1", "var2", "var23", "_task_queue", "_profile"])
+        self.assertSequenceEqual(list(form.fields.keys()), ["var1", "var2", "var23", "_profile", "_job_queue"])
 
     def test_no_field_order(self):
         """
@@ -81,7 +81,7 @@ def test_no_field_order(self):
         name = "TestNoFieldOrder"
         job_class = get_job(f"{module}.{name}")
         form = job_class().as_form()
-        self.assertSequenceEqual(list(form.fields.keys()), ["var23", "var2", "_task_queue", "_profile"])
+        self.assertSequenceEqual(list(form.fields.keys()), ["var23", "var2", "_profile", "_job_queue"])
 
     def test_no_field_order_inherited_variable(self):
         """
@@ -93,7 +93,7 @@ def test_no_field_order_inherited_variable(self):
         form = job_class().as_form()
         self.assertSequenceEqual(
             list(form.fields.keys()),
-            ["testvar1", "b_testvar2", "a_testvar3", "_task_queue", "_profile"],
+            ["testvar1", "b_testvar2", "a_testvar3", "_profile", "_job_queue"],
         )
 
     def test_dryrun_default(self):
@@ -115,48 +115,6 @@ def test_dryrun_default(self):
         form = job_class().as_form()
         self.assertEqual(form.fields["dryrun"].initial, job_model.dryrun_default)
 
-    @mock.patch("nautobot.extras.utils.get_celery_queues")
-    def test_job_class_task_queues(self, mock_get_celery_queues):
-        """
-        Test job form with custom task queues defined on the job class
-        """
-        module = "task_queues"
-        name = "TestWorkerQueues"
-        mock_get_celery_queues.return_value = {"celery": 4, "irrelevant": 5}
-        job_class, _ = get_job_class_and_model(module, name)
-        form = job_class().as_form()
-        self.assertInHTML(
-            """<tr><th><label for="id__task_queue">Task queue:</label></th>
-            <td><select name="_task_queue" class="form-control" placeholder="Task queue" id="id__task_queue">
-            <option value="celery">celery (4 workers)</option>
-            <option value="nonexistent">nonexistent (0 workers)</option></select><br>
-            <span class="helptext">The task queue to route this job to</span>
-            <input type="hidden" name="_profile" value="False" id="id__profile"></td></tr>""",
-            form.as_table(),
-        )
-
-    @mock.patch("nautobot.extras.utils.get_celery_queues")
-    def test_job_class_task_queues_override(self, mock_get_celery_queues):
-        """
-        Test job form with custom task queues defined on the job class and overridden on the model
-        """
-        module = "task_queues"
-        name = "TestWorkerQueues"
-        mock_get_celery_queues.return_value = {"default": 1, "irrelevant": 5}
-        job_class, job_model = get_job_class_and_model(module, name)
-        job_model.task_queues = ["default", "priority"]
-        job_model.task_queues_override = True
-        job_model.save()
-        form = job_class().as_form()
-        self.assertInHTML(
-            """<tr><th><label for="id__task_queue">Task queue:</label></th>
-            <td><select name="_task_queue" class="form-control" placeholder="Task queue" id="id__task_queue">
-            <option value="default">default (1 worker)</option>
-            <option value="priority">priority (0 workers)</option>
-            </select><br><span class="helptext">The task queue to route this job to</span>
-            <input type="hidden" name="_profile" value="False" id="id__profile"></td></tr>""",
-            form.as_table(),
-        )
 
     def test_supports_dryrun(self):
         """
@@ -891,7 +849,7 @@ def test_form_field(self):
         job_class, _job_model = get_job_class_and_model(module, name)
         form = job_class().as_form()
         self.assertSequenceEqual(
-            list(form.fields.keys()), ["object_pk", "object_model_name", "_task_queue", "_profile"]
+            list(form.fields.keys()), ["object_pk", "object_model_name", "_profile", "_job_queue"]
         )
 
     def test_hidden(self):
@@ -955,7 +913,7 @@ def test_form_field(self):
         name = "TestJobHookReceiverLog"
         job_class, _job_model = get_job_class_and_model(module, name)
         form = job_class().as_form()
-        self.assertSequenceEqual(list(form.fields.keys()), ["object_change", "_task_queue", "_profile"])
+        self.assertSequenceEqual(list(form.fields.keys()), ["object_change", "_profile", "_job_queue"])
 
     def test_hidden(self):
         module = "job_hook_receiver"

nautobot/extras/tests/test_utils.py

@@ -1,5 +1,7 @@
 from unittest import mock
 
+from django.core.cache import cache
+
 from nautobot.core.testing import TestCase
 from nautobot.extras.registry import registry
 from nautobot.extras.utils import get_celery_queues, get_worker_count, populate_model_features_registry
@@ -17,20 +19,23 @@ def test_get_celery_queues(self, mock_active_queues):
             self.assertDictEqual(get_celery_queues(), {"queue1": 1})
 
         with self.subTest("2 workers 2 shared queues"):
+            cache.clear()
             mock_active_queues.return_value = {
                 "celery@worker1": [{"name": "queue1"}, {"name": "queue2"}],
                 "celery@worker2": [{"name": "queue1"}, {"name": "queue2"}],
             }
             self.assertDictEqual(get_celery_queues(), {"queue1": 2, "queue2": 2})
 
         with self.subTest("2 workers 2 individual queues"):
+            cache.clear()
             mock_active_queues.return_value = {
                 "celery@worker1": [{"name": "queue1"}],
                 "celery@worker2": [{"name": "queue2"}],
             }
             self.assertDictEqual(get_celery_queues(), {"queue1": 1, "queue2": 1})
 
         with self.subTest("2 workers 3 queues"):
+            cache.clear()
             mock_active_queues.return_value = {
                 "celery@worker1": [{"name": "queue1"}, {"name": "queue3"}],
                 "celery@worker2": [{"name": "queue2"}],

nautobot/extras/tests/test_views.py

@@ -2338,7 +2338,7 @@ def setUpTestData(cls):
         cls.data_run_immediately = {
             "_schedule_type": "immediately",
         }
-
+        job_queues = JobQueue.objects.all()[:3]
         cls.form_data = {
             "enabled": True,
             "grouping_override": True,
@@ -2359,8 +2359,7 @@ def setUpTestData(cls):
             "time_limit": 650,
             "has_sensitive_variables": False,
             "has_sensitive_variables_override": True,
-            "task_queues": "overridden,priority",
-            "task_queues_override": True,
+            "job_queues": [queue.pk for queue in job_queues],
         }
         # This form is emulating the non-conventional JobBulkEditForm
         cls.bulk_edit_data = {
@@ -2381,8 +2380,7 @@ def setUpTestData(cls):
             "time_limit": "",
             "has_sensitive_variables": False,
             "clear_has_sensitive_variables_override": False,
-            "task_queues": "overridden,priority",
-            "clear_task_queues_override": False,
+            "job_queues": [queue.pk for queue in job_queues],
         }
 
     def validate_job_data_after_bulk_edit(self, pk_list, old_data):
@@ -2597,13 +2595,14 @@ def test_rerun_job(self, mock_task_queues_as_choices):
         self.add_permissions("extras.view_jobresult")
 
         mock_task_queues_as_choices.return_value = [("default", ""), ("queue1", ""), ("uniquequeue", "")]
+        job_queue = JobQueue.objects.create(name="uniquequeue", queue_type=JobQueueTypeChoices.TYPE_CELERY)
         job_celery_kwargs = {
             "nautobot_job_job_model_id": self.test_required_args.id,
             "nautobot_job_profile": True,
             "nautobot_job_user_id": self.user.id,
-            "queue": "uniquequeue",
+            "queue": job_queue.pk,
         }
-
+        self.test_required_args.job_queues.set([job_queue])
         previous_result = JobResult.objects.create(
             job_model=self.test_required_args,
             user=self.user,
@@ -2614,8 +2613,7 @@ def test_rerun_job(self, mock_task_queues_as_choices):
         run_url = reverse("extras:job_run", kwargs={"pk": self.test_required_args.pk})
         response = self.client.get(f"{run_url}?kwargs_from_job_result={previous_result.pk!s}")
         content = extract_page_body(response.content.decode(response.charset))
-
-        self.assertInHTML('<option value="uniquequeue" selected>', content)
+        self.assertInHTML(f'<option value="{job_queue.pk}" selected>{job_queue}</option>', content)
         self.assertInHTML(
             '<input type="text" name="var" value="456" class="form-control" required placeholder="None" id="id_var">',
             content,
@@ -2736,7 +2734,7 @@ def test_run_job_with_invalid_task_queue(self, _):
 
         data = {
             "_schedule_type": "immediately",
-            "_task_queue": "invalid",
+            "job_queue": "invalid",
         }
 
         for run_url in self.run_urls:
@@ -2746,7 +2744,7 @@ def test_run_job_with_invalid_task_queue(self, _):
             errors = extract_form_failures(response.content.decode(response.charset))
             self.assertEqual(
                 errors,
-                ["_task_queue: Select a valid choice. invalid is not one of the available choices."],
+                ["job_queue: Select a valid choice. invalid is not one of the available choices."],
             )
 
     @mock.patch("nautobot.extras.views.get_worker_count", return_value=1)
@@ -2911,14 +2909,15 @@ def test_task_queue_hidden_input_is_present(self):
         response = self.client.get(self.location_type.get_absolute_url(), follow=True)
         self.assertEqual(response.status_code, 200)
         content = extract_page_body(response.content.decode(response.charset))
-        self.assertIn(f'<input type="hidden" name="_task_queue" value="{self.job.task_queues[0]}">', content, content)
+        job_queues = self.job.job_queues.all().values_list("name", flat=True)
+        self.assertIn(f'<input type="hidden" name="_job_queue" value="{job_queues[0]}">', content, content)
         self.job.task_queues_override = False
         self.job.save()
         response = self.client.get(self.location_type.get_absolute_url(), follow=True)
         self.assertEqual(response.status_code, 200)
         content = extract_page_body(response.content.decode(response.charset))
         self.assertIn(
-            f'<input type="hidden" name="_task_queue" value="{settings.CELERY_TASK_DEFAULT_QUEUE}">', content, content
+            f'<input type="hidden" name="_job_queue" value="{settings.CELERY_TASK_DEFAULT_QUEUE}">', content, content
         )
 
     def test_view_object_with_unsafe_text(self):

nautobot/extras/utils.py

@@ -371,6 +371,7 @@ def get_celery_queues():
         celery_queues = cache.get("nautobot.extras.utils.get_celery_queues")
 
     if celery_queues is None:
+        celery_queues = {}
         celery_inspect = app.control.inspect()
         active_queues = celery_inspect.active_queues()
         if active_queues is None:

output.txt

@@ -0,0 +1,60 @@
+Running docker compose command "ps --services --filter status=running"
+Running docker compose command "exec nautobot coverage erase"
+PYTHON_VER=3.11 \
+docker compose \
+    --project-name "nautobot" \
+    --project-directory "/Users/hanlinmiao/Desktop/nautobot/development/" \
+    -f "/Users/hanlinmiao/Desktop/nautobot/development/docker-compose.yml" \
+    -f "/Users/hanlinmiao/Desktop/nautobot/development/docker-compose.postgres.yml" \
+    -f "/Users/hanlinmiao/Desktop/nautobot/development/docker-compose.dev.yml" exec nautobot coverage erase
+Running docker compose command "ps --services --filter status=running"
+Running docker compose command "exec nautobot coverage run --module nautobot.core.cli test nautobot.extras.tests.test_views.JobTestCase 
+--config=nautobot/core/tests/nautobot_config.py --cache-test-fixtures --keepdb"
+PYTHON_VER=3.11 \
+docker compose \
+    --project-name "nautobot" \
+    --project-directory "/Users/hanlinmiao/Desktop/nautobot/development/" \
+    -f "/Users/hanlinmiao/Desktop/nautobot/development/docker-compose.yml" \
+    -f "/Users/hanlinmiao/Desktop/nautobot/development/docker-compose.postgres.yml" \
+    -f "/Users/hanlinmiao/Desktop/nautobot/development/docker-compose.dev.yml" exec nautobot coverage run \
+    --module nautobot.core.cli test nautobot.extras.tests.test_views.JobTestCase \
+    --config=nautobot/core/tests/nautobot_config.py \
+    --cache-test-fixtures \
+    --keepdb
+Using NautobotPerformanceTestRunner to run tests ...
+Found 52 test(s).
+Using existing test database for alias 'default'...
+Flushing all existing data from the database "default"...
+Loading factory data from file development/factory_dump.c1cd4b5d9b392126e57f.json
+Installed 8587 object(s) from 1 fixture(s)
+Database default populated successfully!
+System check identified some issues:
+
+WARNINGS:
+<Settings "nautobot_config">: (nautobot.core.W005) STORAGE_CONFIG has been set but STORAGE_BACKEND is not defined. STORAGE_CONFIG will be ignored.
+
+System check identified 1 issue (0 silenced).
+...................................F...............s
+======================================================================
+FAIL: test_run_job_with_invalid_task_queue (nautobot.extras.tests.test_views.JobTestCase.test_run_job_with_invalid_task_queue)
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.11/unittest/mock.py", line 1378, in patched
+    return func(*newargs, **newkeywargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/source/nautobot/extras/tests/test_views.py", line 2742, in test_run_job_with_invalid_task_queue
+    self.assertHttpStatus(response, 200, msg=run_url)
+  File "/source/nautobot/core/testing/mixins.py", line 179, in assertHttpStatus
+    self.assertIn(response.status_code, expected_status, err_message)
+AssertionError: 302 not found in [200] : /extras/jobs/pass.TestPass/run/
+Expected HTTP status(es) [200]; received 302:
+No data
+
+----------------------------------------------------------------------
+Ran 52 tests in 40.897s
+
+FAILED (failures=1, skipped=1)
+Emptying test database "default"...
+Database default emptied!
+Preserving test database for alias 'default'...
+