From 0c6b27f4d12f99b1e365f71717c9b20d1a415601 Mon Sep 17 00:00:00 2001 From: Marlon Keating Date: Mon, 11 Dec 2023 19:35:09 +0000 Subject: [PATCH] feat: Add manufacture_data django command test: Fix test configuration test: improve test coverage test: case for nonstandard model casing test: more coverage docs: Add documentation for manufacture_data --- CHANGELOG.rst | 7 + README.rst | 7 +- edx_django_utils/data_generation/README.rst | 74 ++++ edx_django_utils/data_generation/__init__.py | 0 .../data_generation/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/manufacture_data.py | 374 ++++++++++++++++++ .../data_generation/tests/__init__.py | 5 + .../data_generation/tests/apps.py | 9 + .../data_generation/tests/factories.py | 39 ++ .../data_generation/tests/models.py | 34 ++ .../data_generation/tests/test_management.py | 338 ++++++++++++++++ edx_django_utils/tests/__init__.py | 0 requirements/base.txt | 2 +- requirements/dev.txt | 91 +---- requirements/doc.in | 4 +- requirements/doc.txt | 20 +- requirements/quality.txt | 75 +--- requirements/test.in | 1 + requirements/test.txt | 11 +- test_settings.py | 2 + 21 files changed, 957 insertions(+), 136 deletions(-) create mode 100644 edx_django_utils/data_generation/README.rst create mode 100644 edx_django_utils/data_generation/__init__.py create mode 100644 edx_django_utils/data_generation/management/__init__.py create mode 100644 edx_django_utils/data_generation/management/commands/__init__.py create mode 100644 edx_django_utils/data_generation/management/commands/manufacture_data.py create mode 100644 edx_django_utils/data_generation/tests/__init__.py create mode 100644 edx_django_utils/data_generation/tests/apps.py create mode 100644 edx_django_utils/data_generation/tests/factories.py create mode 100644 edx_django_utils/data_generation/tests/models.py create mode 100644 edx_django_utils/data_generation/tests/test_management.py create mode 100644 edx_django_utils/tests/__init__.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index bf6d209c..d85ef113 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -11,6 +11,13 @@ Change Log .. There should always be an "Unreleased" section for changes pending release. +[5.10.0] - 2023-12-11 +-------------------- + +Added +~~~~~ +* manufacture_data management command + [5.9.0] - 2023-11-27 -------------------- diff --git a/README.rst b/README.rst index 9c75ae94..c8abb858 100644 --- a/README.rst +++ b/README.rst @@ -23,13 +23,12 @@ This repository includes shared utilities for: * `Logging Utilities`_: Includes log filters and an encrypted logging helper. -* `Monitoring Utilities`_: Includes Middleware and utilities for enhanced monitoring. - At this time, supports NewRelic monitoring. - * `Plugin Infrastructure`_: Enables enhanced Django Plugin capabilities. * `Security Utilities`_: Includes a middleware to add CSP response headers. +* `Data Generation`_: Management command for generating Django data based on model factories. + .. _Cache Utilities: edx_django_utils/cache/README.rst .. _Django User and Group Utilities: edx_django_utils/user/README.rst @@ -44,6 +43,8 @@ This repository includes shared utilities for: .. _Security Utilities: edx_django_utils/security/README.rst +.. _Data Generation: edx_django_utils/data_generation/README.rst + Documentation ------------- diff --git a/edx_django_utils/data_generation/README.rst b/edx_django_utils/data_generation/README.rst new file mode 100644 index 00000000..0067cc39 --- /dev/null +++ b/edx_django_utils/data_generation/README.rst @@ -0,0 +1,74 @@ +Django Data Generation +###################### + + +Setup +===== +Add 'edx_django_utils.data_generation' to `INSTALLED_APPS` + +Usage +===== + +(Using https://github.com/openedx/edx-enterprise/blob/master/enterprise/models.py through Devstack as an example) + +Generating Basic Model +---------------------- +Upon invoking the command, supply a model param (--model) that is a complete path to a model that has a corresponding test factory: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomer` + +This will generate an enterprise customer record with place holder values according to the test factory + +Customizing Model Values +------------------------ +We can also provide customizations to the record being generated: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomer --name "FRED"` + + 'EnterpriseCustomer' fields: {'name': 'FRED'} + +We can supply parent model/subfactory customizations as well: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerCatalog --enterprise_customer__site__name "Fred" --enterprise_catalog_query__title "JOE SHMO" --title "who?"` + + 'EnterpriseCustomerCatalog' fields: {'title': 'who?'} + 'EnterpriseCustomer' fields: {} + 'Site' fields: {'name': 'Fred'} + + 'EnterpriseCatalogQuery' fields: {'title': 'JOE SHMO'} + +Note the non subclass customization --title "who?" is applied to the specified model EnterpriseCustomerCatalog + +Customizing Foreign Keys +------------------------ +Say we want to supply an existing record as a FK to our object: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer 994599e6-3787-48ba-a2d1-42d1bdf6c46e` + + 'EnterpriseCustomerUser' fields: {} + 'EnterpriseCustomer' PK: 994599e6-3787-48ba-a2d1-42d1bdf6c46e + +or we can do something like: +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer__site 9 --enterprise_customer__name "joe"` + + 'EnterpriseCustomerUser' fields: {} + 'EnterpriseCustomer' fields: {'name': 'joe'} + 'Site' PK: 9 + +Unsupported Cases +----------------- +One limitation of this script is that it can only fetch or customize, you cannot customize a specified, existing FK: +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer__site__name "fred" --enterprise_customer 994599e6-3787-48ba-a2d1-42d1bdf6c46e` + +which would yield a +`CommandError: This script does not support customizing provided existing objects` + +Error Cases +----------- + +If you try and get something that doesn't exist: + +`./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer ` + +we'd get: +`CommandError: Provided FK value: does not exist on EnterpriseCustomer` diff --git a/edx_django_utils/data_generation/__init__.py b/edx_django_utils/data_generation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_django_utils/data_generation/management/__init__.py b/edx_django_utils/data_generation/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_django_utils/data_generation/management/commands/__init__.py b/edx_django_utils/data_generation/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/edx_django_utils/data_generation/management/commands/manufacture_data.py b/edx_django_utils/data_generation/management/commands/manufacture_data.py new file mode 100644 index 00000000..377ab0a4 --- /dev/null +++ b/edx_django_utils/data_generation/management/commands/manufacture_data.py @@ -0,0 +1,374 @@ +""" +Management command for making things with test factories + +Arguments +======== + +--model: complete path to a model that has a corresponding test factory +--{model_attribute}: (Optional) Value of a model's attribute that will override test factory's default attribute value +--{model_foreignkey__foreignkey_attribute}: (Optional) Value of a model's attribute + that will override test factory's default attribute value + + +Examples +======== + +./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomer +This will generate an enterprise customer record with placeholder values according to the test factory + +./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomer --name "FRED" +will produce the customized record: +'EnterpriseCustomer' fields: {'name': 'FRED'} + +./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerCatalog / + --enterprise_customer__site__name "Fred" --enterprise_catalog_query__title "JOE SHMO" --title "who?" +will result in: +'EnterpriseCustomerCatalog' fields: {'title': 'who?'} + 'EnterpriseCustomer' fields: {} + 'Site' fields: {'name': 'Fred'} + 'EnterpriseCatalogQuery' fields: {'title': 'JOE SHMO'} + +To supply an existing record as a FK to our object: +./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser / + --enterprise_customer 994599e6-3787-48ba-a2d1-42d1bdf6c46e +'EnterpriseCustomerUser' fields: {} + 'EnterpriseCustomer' PK: 994599e6-3787-48ba-a2d1-42d1bdf6c46e + +or we can do something like: +./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser / + --enterprise_customer__site 9 --enterprise_customer__name "joe" +which would yield: +'EnterpriseCustomerUser' fields: {} + 'EnterpriseCustomer' fields: {'name': 'joe'} + 'Site' PK: 9 + + +Errors +====== + +But if you try and get something that doesn't exist... +./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser --enterprise_customer +we'd get: +CommandError: Provided FK value: does not exist on EnterpriseCustomer + +Another limitation of this script is that it can only fetch or customize, you cannot customize a specified, existing FK + ./manage.py lms manufacture_data --model enterprise.models.EnterpriseCustomerUser / + --enterprise_customer__site__name "fred" --enterprise_customer 994599e6-3787-48ba-a2d1-42d1bdf6c46e +would yield CommandError: This script does not support customizing provided existing objects +""" + +import logging +import re +import sys + +import factory +from django.core.exceptions import ImproperlyConfigured +from django.core.management.base import BaseCommand, CommandError, SystemCheckError, handle_default_options +from django.db import connections +from factory.declarations import SubFactory + +log = logging.getLogger(__name__) + +def is_not_pascal(string): + """ + helper method to detect if strings are not Pascal case. + """ + return '_' in string or string.islower() + + +def convert_to_pascal_if_needed(string): + """ + helper method to convert strings to Pascal case. + """ + if is_not_pascal(string): + return string.replace("_", " ").title().replace(" ", "") + else: + return string + + +def pairwise(iterable): + """ + Convert a list into a list of tuples of adjacent elements. + s -> [ (s0, s1), (s2, s3), (s4, s5), ... ] + """ + a = iter(iterable) + return zip(a, a) + + +def all_subclasses(cls): + """ + Recursively get all subclasses of a class + https://stackoverflow.com/a/3862957 + """ + return set(cls.__subclasses__()).union( + [s for c in cls.__subclasses__() for s in all_subclasses(c)]) + + +def convert_to_snake(string): + """ + Helper method to convert strings to snake case. + """ + return re.sub(r'(?' + + +def build_tree_from_field_list(list_of_fields, provided_factory, base_node, customization_value): + """ + Builds a non-binary tree of nodes based on a list of children nodes, using a base node and it's associated data + factory as the parent node the user provided value as a reference to a potential, existing record. + + - list_of_fields (list of strings): the linked list of associated objects to create. Example- + ['enterprise_customer_user', 'enterprise_customer', 'site'] + - provided_factory (factory.django.DjangoModelFactory): The data factory of the base_node. + - base_node (Node): The parent node of the desired tree to build. + - customization_value (string): The value to be assigned to the object associated with the last value in the + ``list_of_fields`` param. Can either be a FK if the last value is a subfactory, or alternatively + a custom value to be assigned to the field. Example- + list_of_fields = ['enterprise_customer_user', 'enterprise_customer', 'site'], + customization_value = 9 + or + list_of_fields = ['enterprise_customer_user', 'enterprise_customer', 'name'], + customization_value = "FRED" + """ + current_factory = provided_factory + current_node = base_node + for index, value in enumerate(list_of_fields): + try: + # First we need to figure out if the current field is a sub factory or not + f = getattr(current_factory, value) + if isinstance(f, SubFactory): + fk_object = None + f_model = f.get_factory()._meta.get_model_class() + + # if we're at the end of the list + if index == len(list_of_fields) - 1: + # verify that the provided customization value is a valid pk for the model + try: + fk_object = f_model.objects.get(pk=customization_value) + except f_model.DoesNotExist as exc: + raise CommandError( + f"Provided FK value: {customization_value} does not exist on {f_model.__name__}" + ) from exc + + # Look for the node in the tree + if node := current_node.find_value(f_model.__name__): + # Not supporting customizations and FK's + if (bool(node.customizations) or bool(node.children)) and bool(fk_object): + raise CommandError("This script does not support customizing provided existing objects") + # If we found the valid FK earlier, assign it to the node + if fk_object: + node.instance = fk_object + # Add the field to the children of the current node + if node not in current_node.children: + current_node.add_child(node) + # Set current node and move on + current_node = node + else: + # Create a new node + node = Node( + f_model.__name__, + ) + node.factory = f.get_factory() + # If we found the valid FK earlier, assign it to the node + if fk_object: + node.instance = fk_object + # Add the field to the children of the current node + current_node.add_child(node) + + current_node = node + current_factory = f.get_factory() + else: + if current_node.instance: + raise CommandError("This script cannot modify existing objects") + current_node.set_single_customization(value, customization_value) + except AttributeError as exc: + log.error(f'Could not find value: {value} in factory: {current_factory}') + raise CommandError(f'Could not find value: {value} in factory: {current_factory}') from exc + return base_node + + +class Command(BaseCommand): + """ + Management command for generating Django records from factories with custom attributes + + Example usage: + $ ./manage.py manufacture_data --model enterprise.models.enterprise_customer \ + --name "Test Enterprise" --slug "test-enterprise" + """ + + def add_arguments(self, parser): + parser.add_argument( + '--model', + dest='model', + help='The model for which the record will be written', + ) + + def run_from_argv(self, argv): + """ + Re-implemented from https://github.com/django/django/blob/main/django/core/management/base.py#L395 in order to + support individual field customization. We will need to keep this method up to date with our current version of + Django BaseCommand. + + Uses ``parse_known_args`` instead of ``parse_args`` to not throw an error when encountering unknown arguments + + https://docs.python.org/3.8/library/argparse.html#argparse.ArgumentParser.parse_known_args + """ + self._called_from_command_line = True + parser = self.create_parser(argv[0], argv[1]) + options, unknown = parser.parse_known_args(argv[2:]) + + # Add the unknowns into the options for use of the handle method + paired_unknowns = pairwise(unknown) + field_customizations = {} + for field, value in paired_unknowns: + field_customizations[field.strip("--")] = value + options.field_customizations = field_customizations + + cmd_options = vars(options) + # Move positional args out of options to mimic legacy optparse + args = cmd_options.pop("args", ()) + handle_default_options(options) + try: + self.execute(*args, **cmd_options) + except CommandError as e: + if options.traceback: + raise + + # SystemCheckError takes care of its own formatting. + if isinstance(e, SystemCheckError): + self.stderr.write(str(e), lambda x: x) + else: + self.stderr.write("%s: %s" % (e.__class__.__name__, e)) + sys.exit(e.returncode) + finally: + try: + connections.close_all() + except ImproperlyConfigured: + # Ignore if connections aren't setup at this point (e.g. no + # configured settings). + pass + + def handle(self, *args, **options): + """ + Entry point for management command execution. + """ + if not options.get('model'): + log.error("Did not receive a model") + raise CommandError("Did not receive a model") + # Convert to Pascal case if the provided name is snake case/is all lowercase + path_of_model = options.get('model').split(".") + last_path = convert_to_pascal_if_needed(path_of_model[-1]) + + provided_model = '.'.join(path_of_model[:-1]) + '.' + last_path + # Get all installed/imported factories + factories_list = all_subclasses(factory.django.DjangoModelFactory) + # Find the factory that matches the provided model + for potential_factory in factories_list: + # Fetch the model for the factory + factory_model = potential_factory._meta.model + # Check if the factories model matches the provided model + if f"{factory_model.__module__}.{convert_to_pascal_if_needed(factory_model.__name__)}" == provided_model: + # Now that we have the right factory, we can build according to the provided custom attributes + field_customizations = options.get('field_customizations', {}) + base_node = Node(factory_model.__name__) + base_node.factory = potential_factory + # For each provided custom attribute... + for field, value in field_customizations.items(): + + # We need to build a tree of objects to be created and may be customized by other custom attributes + stripped_field = field.strip("--") + fk_field_customization_split = stripped_field.split("__") + base_node = build_tree_from_field_list( + fk_field_customization_split, + potential_factory, + base_node, + value, + ) + + built_node = base_node.build_records() + log.info(f"\nGenerated factory data: \n{base_node}") + return str(list(built_node.values())[0].pk) + + log.error(f"Provided model: {provided_model} does not exist or does not have an associated factory") + raise CommandError(f"Provided model: {provided_model}'s factory is not imported or does not exist") diff --git a/edx_django_utils/data_generation/tests/__init__.py b/edx_django_utils/data_generation/tests/__init__.py new file mode 100644 index 00000000..0fa7ac0d --- /dev/null +++ b/edx_django_utils/data_generation/tests/__init__.py @@ -0,0 +1,5 @@ +""" +Tests for Data Generation +""" + +default_app_config = 'edx_django_utils.data_generation.tests.apps.DataGenerationTestsConfig' diff --git a/edx_django_utils/data_generation/tests/apps.py b/edx_django_utils/data_generation/tests/apps.py new file mode 100644 index 00000000..0b177c49 --- /dev/null +++ b/edx_django_utils/data_generation/tests/apps.py @@ -0,0 +1,9 @@ +""" +Tests for Data Generation +""" +from django.apps import AppConfig + +class DataGenerationTestsConfig(AppConfig): + name = 'edx_django_utils.data_generation.tests' + label = 'data_generation_tests' # Needed to avoid App label duplication with other tests modules + \ No newline at end of file diff --git a/edx_django_utils/data_generation/tests/factories.py b/edx_django_utils/data_generation/tests/factories.py new file mode 100644 index 00000000..59524f5a --- /dev/null +++ b/edx_django_utils/data_generation/tests/factories.py @@ -0,0 +1,39 @@ +""" +Factories for models used in testing manufacture_data command +""" + +import factory + +from edx_django_utils.data_generation.tests.models import TestPerson, TestPersonContactInfo, test_model_nonstandard_casing + + +class TestPersonFactory(factory.django.DjangoModelFactory): + """ + Test Factory for TestPerson + """ + class Meta: + model = TestPerson + + first_name = 'John' + last_name = 'Doe' + + +class TestPersonContactInfoFactory(factory.django.DjangoModelFactory): + """ + Test Factory for TestPersonContactInfo + """ + class Meta: + model = TestPersonContactInfo + + test_person = factory.SubFactory(TestPersonFactory) + address = '123 4th st, Fiveville, AZ, 67890' + + +class TestModelNonstandardCasingFactory(factory.django.DjangoModelFactory): + """ + Test Factory for test_model_nonstandard_casing + """ + class Meta: + model = test_model_nonstandard_casing + + test_field = 'TEST' diff --git a/edx_django_utils/data_generation/tests/models.py b/edx_django_utils/data_generation/tests/models.py new file mode 100644 index 00000000..83ac9657 --- /dev/null +++ b/edx_django_utils/data_generation/tests/models.py @@ -0,0 +1,34 @@ +""" +Models used in testing manufacture_data command +""" +from django.db import models + + +class TestPerson(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + + first_name = models.CharField(max_length=30) + last_name = models.CharField(max_length=30) + + +class TestPersonContactInfo(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + test_person = models.ForeignKey(TestPerson, on_delete=models.CASCADE) + address = models.CharField(max_length=100) + + +class test_model_nonstandard_casing(models.Model): + """ + For use in testing manufacture_data command + """ + class Meta: + app_label = 'data_generation_tests' + test_field = models.CharField(max_length=30) diff --git a/edx_django_utils/data_generation/tests/test_management.py b/edx_django_utils/data_generation/tests/test_management.py new file mode 100644 index 00000000..b467f622 --- /dev/null +++ b/edx_django_utils/data_generation/tests/test_management.py @@ -0,0 +1,338 @@ +""" +Test management commands and related functions. +""" + +from argparse import _AppendConstAction, _CountAction, _StoreConstAction, _SubParsersAction + +from django.core.management import get_commands, load_command_class +from django.core.management.base import BaseCommand, CommandError +from django.test import TestCase +from pytest import mark + +from edx_django_utils.data_generation.management.commands.manufacture_data import Command, Node +# pylint: disable=unused-import +from edx_django_utils.data_generation.tests.factories import TestPersonContactInfoFactory, TestPersonFactory, TestModelNonstandardCasingFactory +from edx_django_utils.data_generation.tests.models import TestPerson, TestPersonContactInfo, test_model_nonstandard_casing + +class TestCommand(Command): + """ + Class for use in testing manufacture_data command via run_from_argv + """ + def check(self, *args): + # Skip checks that aren't needed or configured in test suite + pass + + +# Copied from django.core.management.__init__.py +# https://github.com/django/django/blob/1ad7761ee616341295f36c80f78b86ff79d5b513/django/core/management/__init__.py#L83 +def call_command(command_name, *args, **options): + """ + Call the given command, with the given options and args/kwargs. + + This is the primary API you should use for calling specific commands. + + `command_name` may be a string or a command object. Using a string is + preferred unless the command object is required for further processing or + testing. + + Some examples: + call_command('migrate') + call_command('shell', plain=True) + call_command('sqlmigrate', 'myapp') + + from django.core.management.commands import flush + cmd = flush.Command() + call_command(cmd, verbosity=0, interactive=False) + # Do something with cmd ... + """ + if isinstance(command_name, BaseCommand): + # Command object passed in. + command = command_name + command_name = command.__class__.__module__.split(".")[-1] + else: + # Load the command object by name. + try: + app_name = get_commands()[command_name] + except KeyError: + raise CommandError("Unknown command: %r" % command_name) # pylint: disable=raise-missing-from + + if isinstance(app_name, BaseCommand): + # If the command is already loaded, use it directly. + command = app_name + else: + command = load_command_class(app_name, command_name) + + # Simulate argument parsing to get the option defaults (see #10080 for details). + parser = command.create_parser("", command_name) + # Use the `dest` option name from the parser option + opt_mapping = { + min(s_opt.option_strings).lstrip("-").replace("-", "_"): s_opt.dest + for s_opt in parser._actions # pylint: disable=protected-access + if s_opt.option_strings + } + arg_options = {opt_mapping.get(key, key): value for key, value in options.items()} + parse_args = [] + for arg in args: + if isinstance(arg, (list, tuple)): + parse_args += map(str, arg) + else: + parse_args.append(str(arg)) + + def get_actions(parser): + # Parser actions and actions from sub-parser choices. + for opt in parser._actions: # pylint: disable=protected-access + if isinstance(opt, _SubParsersAction): + for sub_opt in opt.choices.values(): + yield from get_actions(sub_opt) + else: + yield opt + + parser_actions = list(get_actions(parser)) + mutually_exclusive_required_options = { + opt + for group in parser._mutually_exclusive_groups # pylint: disable=protected-access + for opt in group._group_actions # pylint: disable=protected-access + if group.required + } + # Any required arguments which are passed in via **options must be passed + # to parse_args(). + for opt in parser_actions: + if opt.dest in options and ( + opt.required or opt in mutually_exclusive_required_options + ): + opt_dest_count = sum(v == opt.dest for v in opt_mapping.values()) + if opt_dest_count > 1: + raise TypeError( + f"Cannot pass the dest {opt.dest!r} that matches multiple " + f"arguments via **options." + ) + parse_args.append(min(opt.option_strings)) + if isinstance(opt, (_AppendConstAction, _CountAction, _StoreConstAction)): + continue + value = arg_options[opt.dest] + if isinstance(value, (list, tuple)): + parse_args += map(str, value) + else: + parse_args.append(str(value)) + defaults = parser.parse_args(args=parse_args) + + defaults = dict(defaults._get_kwargs(), **arg_options) # pylint: disable=protected-access + # Commented out section allows for unknown options to be passed to the command + + # Raise an error if any unknown options were passed. + # stealth_options = set(command.base_stealth_options + command.stealth_options) + # dest_parameters = {action.dest for action in parser_actions} + # valid_options = (dest_parameters | stealth_options).union(opt_mapping) + # unknown_options = set(options) - valid_options + # if unknown_options: + # raise TypeError( + # "Unknown option(s) for %s command: %s. " + # "Valid options are: %s." + # % ( + # command_name, + # ", ".join(sorted(unknown_options)), + # ", ".join(sorted(valid_options)), + # ) + # ) + # Move positional args out of options to mimic legacy optparse + args = defaults.pop("args", ()) + if "skip_checks" not in options: + defaults["skip_checks"] = True + + return command.execute(*args, **defaults) + +@mark.django_db +class ManufactureDataCommandTests(TestCase): + """ + Test command `manufacture_data`. + """ + command = 'manufacture_data' + + def test_command_requires_model(self): + """ + Test that the manufacture_data command will raise an error if no model is provided. + """ + with self.assertRaises(CommandError): + call_command(self.command) + + def test_command_requires_valid_model(self): + """ + Test that the manufacture_data command will raise an error if the provided model is invalid. + """ + with self.assertRaises(CommandError): + call_command(self.command, model='FakeModel') + + def test_single_object_create_no_customizations(self): + """ + Test that the manufacture_data command will create a single object with no customizations. + """ + assert TestPerson.objects.all().count() == 0 + created_object = call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPerson' + ) + assert TestPerson.objects.all().count() == 1 + assert TestPerson.objects.filter(pk=created_object).exists() + + def test_command_requires_valid_field(self): + """ + Test that the manufacture_data command will raise an error if the provided field is invalid. + """ + with self.assertRaises(CommandError): + call_command( + self.command, + model='TestPerson', + field_customizations={"fake_field": 'fake_value'} + ) + + def test_command_can_customize_fields(self): + """ + Test that the manufacture_data command will create a single object with customizations. + """ + assert TestPerson.objects.all().count() == 0 + created_object = call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPerson', + field_customizations={'first_name': 'Steve'}, + ) + assert TestPerson.objects.all().count() == 1 + assert TestPerson.objects.filter(pk=created_object).exists() + assert TestPerson.objects.filter(pk=created_object).first().first_name == 'Steve' + + def test_command_can_customize_nested_objects(self): + """ + Test that the manufacture_data command supports customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestPersonContactInfo.objects.all().count() == 0 + created_object = call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPersonContactInfo', + field_customizations={'address': '123 4th st', + 'test_person__first_name': 'Joey', 'test_person__last_name': 'Nowhere'}, + ) + assert TestPerson.objects.all().count() == 1 + assert TestPersonContactInfo.objects.all().count() == 1 + assert TestPersonContactInfo.objects.filter( + pk=created_object + ).first().test_person.last_name == 'Nowhere' + + def test_command_cannot_customize_foreign_keys(self): + """ + Error case: customizing nested objects. + Error case: customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestPersonContactInfo.objects.all().count() == 0 + test_person = call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPerson', + field_customizations={'first_name': 'Steve'}, + ) + with self.assertRaises(CommandError): + call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPersonContactInfo', + field_customizations={'address': '123 4th st', + 'test_person': test_person, 'test_person__last_name': "Harvey"}, + ) + + def test_command_object_foreign_key(self): + """ + Test that the manufacture_data command supports creating objects with foreign keys + """ + assert TestPerson.objects.all().count() == 0 + foreign_key_object_id = call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPerson', + field_customizations={'first_name': 'Steve'}, + ) + assert TestPerson.objects.all().count() == 1 + created_object = call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPersonContactInfo', + field_customizations={'test_person': foreign_key_object_id}, + ) + assert TestPersonContactInfo.objects.filter( + pk=created_object + ).first().test_person.first_name == 'Steve' + + def test_argv_command_can_customize_nested_objects(self): + """ + argv: Test that the manufacture_data command supports customizing nested objects. + """ + assert TestPerson.objects.all().count() == 0 + assert TestPersonContactInfo.objects.all().count() == 0 + command = TestCommand() + + command.run_from_argv( + ['manage.py', 'manufacture_data', '--model', + 'edx_django_utils.data_generation.tests.models.TestPersonContactInfo', + '--test_person__last_name', 'Nowhere'] + ) + assert TestPerson.objects.all().count() == 1 + assert TestPersonContactInfo.objects.all().count() == 1 + assert TestPersonContactInfo.objects.first().test_person.last_name == 'Nowhere' + + def test_argv_command_error(self): + """ + argv error: Nested model does not exist + """ + assert TestPerson.objects.all().count() == 0 + assert TestPersonContactInfo.objects.all().count() == 0 + command = TestCommand() + + with self.assertRaises(SystemExit): + command.run_from_argv( + ['manage.py', 'manufacture_data', '--model', + 'edx_django_utils.data_generation.tests.models.ThisModelDoesNotExist'] + ) + + def test_nonstandard_casing(self): + """ + Test that the manufacture_data command will work with models that use non-standard casing + """ + assert test_model_nonstandard_casing.objects.all().count() == 0 + created_object = call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.test_model_nonstandard_casing' + ) + assert test_model_nonstandard_casing.objects.all().count() == 1 + assert test_model_nonstandard_casing.objects.filter(pk=created_object).exists() + + def test_command_nested_nonexistent_model(self): + """ + Error case: Nested model does not exist + """ + with self.assertRaises(CommandError): + call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPersonContactInfo', + field_customizations={'address': '123 4th st', 'test_nonperson__last_name': 'non-name'}, + ) + + def test_command_nested_nonexistent_attribute(self): + """ + Error case: Nested model does not exist + """ + with self.assertRaises(CommandError): + call_command( + self.command, + model='edx_django_utils.data_generation.tests.models.TestPersonContactInfo', + field_customizations={'address': '123 4th st', 'test_person__middle_name': 'Milhaus'} + ) + + def test_node_no_factory(self): + """ + Node error case: no factory provided + """ + node = Node({}) + with self.assertRaises(CommandError): + node.build_records() + + def test_node_repr(self): + """ + Node repr test + """ + node = Node({'data': 'something'}) + assert repr(node) == "" diff --git a/edx_django_utils/tests/__init__.py b/edx_django_utils/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requirements/base.txt b/requirements/base.txt index 1c6b92b1..70ec702c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -18,7 +18,7 @@ django==3.2.23 # django-waffle django-crum==0.7.9 # via -r requirements/base.in -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/base.in newrelic==9.3.0 # via -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index d64a8939..2fbca016 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,9 +5,7 @@ # make upgrade # annotated-types==0.6.0 - # via - # -r requirements/quality.txt - # pydantic + # via pydantic asgiref==3.7.2 # via # -r requirements/quality.txt @@ -25,14 +23,6 @@ cachetools==5.3.2 # via # -r requirements/ci.txt # tox -cerberus==1.3.5 - # via - # -r requirements/quality.txt - # plette -certifi==2023.11.17 - # via - # -r requirements/quality.txt - # requests cffi==1.16.0 # via # -r requirements/quality.txt @@ -42,10 +32,6 @@ chardet==5.2.0 # -r requirements/ci.txt # diff-cover # tox -charset-normalizer==3.3.2 - # via - # -r requirements/quality.txt - # requests click==8.1.7 # via # -r requirements/pip-tools.txt @@ -84,8 +70,6 @@ dill==0.3.7 distlib==0.3.7 # via # -r requirements/ci.txt - # -r requirements/quality.txt - # requirementslib # virtualenv django==3.2.23 # via @@ -96,12 +80,8 @@ django==3.2.23 # edx-i18n-tools django-crum==0.7.9 # via -r requirements/quality.txt -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/quality.txt -docopt==0.6.2 - # via - # -r requirements/quality.txt - # pipreqs edx-i18n-tools==1.3.0 # via -r requirements/dev.in edx-lint==5.3.6 @@ -110,15 +90,17 @@ exceptiongroup==1.2.0 # via # -r requirements/quality.txt # pytest +factory-boy==3.3.0 + # via -r requirements/quality.txt +faker==20.1.0 + # via + # -r requirements/quality.txt + # factory-boy filelock==3.13.1 # via # -r requirements/ci.txt # tox # virtualenv -idna==3.6 - # via - # -r requirements/quality.txt - # requests importlib-metadata==7.0.0 # via # -r requirements/pip-tools.txt @@ -129,7 +111,7 @@ iniconfig==2.0.0 # via # -r requirements/quality.txt # pytest -isort==5.13.0 +isort==5.13.1 # via # -r requirements/quality.txt # pylint @@ -170,33 +152,15 @@ pbr==6.0.0 # via # -r requirements/quality.txt # stevedore -pep517==0.13.1 - # via - # -r requirements/quality.txt - # requirementslib -pip-api==0.0.30 - # via - # -r requirements/quality.txt - # isort pip-tools==7.3.0 # via -r requirements/pip-tools.txt -pipreqs==0.4.13 - # via - # -r requirements/quality.txt - # isort platformdirs==4.1.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint - # requirementslib # tox # virtualenv -plette[validation]==0.4.4 - # via - # -r requirements/quality.txt - # plette - # requirementslib pluggy==1.3.0 # via # -r requirements/ci.txt @@ -215,19 +179,14 @@ pycparser==2.21 # -r requirements/quality.txt # cffi pydantic==2.5.2 - # via - # -r requirements/quality.txt - # inflect - # requirementslib + # via inflect pydantic-core==2.14.5 - # via - # -r requirements/quality.txt - # pydantic + # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.txt pygments==2.17.2 # via diff-cover -pylint==3.0.2 +pylint==3.0.3 # via # -r requirements/quality.txt # edx-lint @@ -267,7 +226,10 @@ pytest-cov==4.1.0 pytest-django==4.7.0 # via -r requirements/quality.txt python-dateutil==2.8.2 - # via -r requirements/dev.in + # via + # -r requirements/dev.in + # -r requirements/quality.txt + # faker python-slugify==8.0.1 # via # -r requirements/quality.txt @@ -281,15 +243,6 @@ pyyaml==6.0.1 # -r requirements/quality.txt # code-annotations # edx-i18n-tools -requests==2.31.0 - # via - # -r requirements/quality.txt - # requirementslib - # yarg -requirementslib==3.0.0 - # via - # -r requirements/quality.txt - # isort six==1.16.0 # via # -r requirements/quality.txt @@ -318,7 +271,6 @@ tomli==2.0.1 # -r requirements/quality.txt # build # coverage - # pep517 # pip-tools # pylint # pyproject-api @@ -328,9 +280,7 @@ tomli==2.0.1 tomlkit==0.12.3 # via # -r requirements/quality.txt - # plette # pylint - # requirementslib tox==4.11.4 # via -r requirements/ci.txt typing-extensions==4.9.0 @@ -339,14 +289,11 @@ typing-extensions==4.9.0 # annotated-types # asgiref # astroid + # faker # inflect # pydantic # pydantic-core # pylint -urllib3==2.1.0 - # via - # -r requirements/quality.txt - # requests virtualenv==20.25.0 # via # -r requirements/ci.txt @@ -355,10 +302,6 @@ wheel==0.42.0 # via # -r requirements/pip-tools.txt # pip-tools -yarg==0.1.9 - # via - # -r requirements/quality.txt - # pipreqs zipp==3.17.0 # via # -r requirements/pip-tools.txt diff --git a/requirements/doc.in b/requirements/doc.in index 4b62d6af..8135a8fd 100644 --- a/requirements/doc.in +++ b/requirements/doc.in @@ -7,4 +7,6 @@ doc8 # reStructuredText style checker sphinx-book-theme # Common theme for all Open edX projects readme_renderer # Validates README.rst for usage on PyPI Sphinx # Documentation builder -twine \ No newline at end of file +twine +factory-boy +pytest #Needed? \ No newline at end of file diff --git a/requirements/doc.txt b/requirements/doc.txt index 54c810e5..32fd1fd3 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -46,7 +46,7 @@ django==3.2.23 # django-waffle django-crum==0.7.9 # via -r requirements/test.txt -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/test.txt doc8==0.11.2 # via @@ -63,6 +63,14 @@ exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest +factory-boy==3.3.0 + # via + # -r requirements/doc.in + # -r requirements/test.txt +faker==20.1.0 + # via + # -r requirements/test.txt + # factory-boy idna==3.6 # via requests imagesize==1.4.1 @@ -137,6 +145,7 @@ pynacl==1.5.0 # via -r requirements/test.txt pytest==7.4.3 # via + # -r requirements/doc.in # -r requirements/test.txt # pytest-cov # pytest-django @@ -144,6 +153,10 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.7.0 # via -r requirements/test.txt +python-dateutil==2.8.2 + # via + # -r requirements/test.txt + # faker pytz==2023.3.post1 # via # -r requirements/test.txt @@ -168,6 +181,10 @@ rich==13.7.0 # via twine secretstorage==3.3.3 # via keyring +six==1.16.0 + # via + # -r requirements/test.txt + # python-dateutil snowballstemmer==2.2.0 # via sphinx soupsieve==2.5 @@ -211,6 +228,7 @@ typing-extensions==4.9.0 # via # -r requirements/test.txt # asgiref + # faker # pydata-sphinx-theme # rich urllib3==2.1.0 diff --git a/requirements/quality.txt b/requirements/quality.txt index 6c282189..cf7e2b8f 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,8 +4,6 @@ # # make upgrade # -annotated-types==0.6.0 - # via pydantic asgiref==3.7.2 # via # -r requirements/test.txt @@ -14,16 +12,10 @@ astroid==3.0.1 # via # pylint # pylint-celery -cerberus==1.3.5 - # via plette -certifi==2023.11.17 - # via requests cffi==1.16.0 # via # -r requirements/test.txt # pynacl -charset-normalizer==3.3.2 - # via requests click==8.1.7 # via # -r requirements/test.txt @@ -43,8 +35,6 @@ ddt==1.7.0 # via -r requirements/test.txt dill==0.3.7 # via pylint -distlib==0.3.7 - # via requirementslib django==3.2.23 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -53,23 +43,25 @@ django==3.2.23 # django-waffle django-crum==0.7.9 # via -r requirements/test.txt -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/test.txt -docopt==0.6.2 - # via pipreqs edx-lint==5.3.6 # via -r requirements/quality.in exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest -idna==3.6 - # via requests +factory-boy==3.3.0 + # via -r requirements/test.txt +faker==20.1.0 + # via + # -r requirements/test.txt + # factory-boy iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -isort==5.13.0 +isort==5.13.1 # via # -r requirements/quality.in # pylint @@ -91,18 +83,8 @@ pbr==6.0.0 # via # -r requirements/test.txt # stevedore -pep517==0.13.1 - # via requirementslib -pip-api==0.0.30 - # via isort -pipreqs==0.4.13 - # via isort platformdirs==4.1.0 - # via - # pylint - # requirementslib -plette[validation]==0.4.4 - # via requirementslib + # via pylint pluggy==1.3.0 # via # -r requirements/test.txt @@ -115,13 +97,9 @@ pycparser==2.21 # via # -r requirements/test.txt # cffi -pydantic==2.5.2 - # via requirementslib -pydantic-core==2.14.5 - # via pydantic pydocstyle==6.3.0 # via -r requirements/quality.in -pylint==3.0.2 +pylint==3.0.3 # via # edx-lint # pylint-celery @@ -146,6 +124,10 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.7.0 # via -r requirements/test.txt +python-dateutil==2.8.2 + # via + # -r requirements/test.txt + # faker python-slugify==8.0.1 # via code-annotations pytz==2023.3.post1 @@ -154,14 +136,11 @@ pytz==2023.3.post1 # django pyyaml==6.0.1 # via code-annotations -requests==2.31.0 - # via - # requirementslib - # yarg -requirementslib==3.0.0 - # via isort six==1.16.0 - # via edx-lint + # via + # -r requirements/test.txt + # edx-lint + # python-dateutil snowballstemmer==2.2.0 # via pydocstyle sqlparse==0.4.4 @@ -178,28 +157,14 @@ tomli==2.0.1 # via # -r requirements/test.txt # coverage - # pep517 # pylint # pytest tomlkit==0.12.3 - # via - # plette - # pylint - # requirementslib + # via pylint typing-extensions==4.9.0 # via # -r requirements/test.txt - # annotated-types # asgiref # astroid - # pydantic - # pydantic-core + # faker # pylint -urllib3==2.1.0 - # via requests -yarg==0.1.9 - # via pipreqs - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/requirements/test.in b/requirements/test.in index 74f0ba9c..60e3b0eb 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -4,6 +4,7 @@ -r base.txt # Core dependencies for this package ddt # Run a test case multiple times with different input +factory_boy # Test factory framework mock # Backport of unittest.mock, available in Python 3.3 pytest-cov # pytest extension for code coverage statistics pytest-django # pytest extension for better Django support diff --git a/requirements/test.txt b/requirements/test.txt index f5e2d612..c08dfcad 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -27,10 +27,14 @@ ddt==1.7.0 # django-waffle django-crum==0.7.9 # via -r requirements/base.txt -django-waffle==4.0.0 +django-waffle==4.1.0 # via -r requirements/base.txt exceptiongroup==1.2.0 # via pytest +factory-boy==3.3.0 + # via -r requirements/test.in +faker==20.1.0 + # via factory-boy iniconfig==2.0.0 # via pytest mock==5.1.0 @@ -61,10 +65,14 @@ pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.7.0 # via -r requirements/test.in +python-dateutil==2.8.2 + # via faker pytz==2023.3.post1 # via # -r requirements/base.txt # django +six==1.16.0 + # via python-dateutil sqlparse==0.4.4 # via # -r requirements/base.txt @@ -79,3 +87,4 @@ typing-extensions==4.9.0 # via # -r requirements/base.txt # asgiref + # faker diff --git a/test_settings.py b/test_settings.py index 9d8e4898..eeb26e30 100644 --- a/test_settings.py +++ b/test_settings.py @@ -41,6 +41,8 @@ def root(*args): "edx_django_utils", "edx_django_utils.admin.tests", "edx_django_utils.user", + 'edx_django_utils.data_generation', + 'edx_django_utils.data_generation.tests', ) LOCALE_PATHS = [root("edx_django_utils", "conf", "locale")]