From 41aad9a8d3e2217fe21717649ba5c2cfffe19f39 Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 21 Mar 2024 23:18:46 +0000 Subject: [PATCH 001/190] Adds new model for DataImportSession --- .../migrations/0023_dataimportsession.py | 29 +++++++++++++++ InvenTree/common/models.py | 36 +++++++++++++++++++ InvenTree/common/status_codes.py | 18 ++++++++++ 3 files changed, 83 insertions(+) create mode 100644 InvenTree/common/migrations/0023_dataimportsession.py create mode 100644 InvenTree/common/status_codes.py diff --git a/InvenTree/common/migrations/0023_dataimportsession.py b/InvenTree/common/migrations/0023_dataimportsession.py new file mode 100644 index 000000000000..00d87b186eff --- /dev/null +++ b/InvenTree/common/migrations/0023_dataimportsession.py @@ -0,0 +1,29 @@ +# Generated by Django 4.2.11 on 2024-03-21 23:12 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + +import common.status_codes + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('common', '0022_projectcode_responsible'), + ] + + operations = [ + migrations.CreateModel( + name='DataImportSession', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data_file', models.FileField(help_text='Data file to import', upload_to='import', verbose_name='Data File')), + ('status', models.PositiveIntegerField(choices=common.status_codes.DataImportStatusCode.items(), default=0, help_text='Import status')), + ('progress', models.PositiveIntegerField(default=0, verbose_name='Progress')), + ('data_columns', models.JSONField(blank=True, null=True, verbose_name='Data Columns')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), + ], + ), + ] diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index 47d9bf47bc8d..20c6dee8abc6 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -41,6 +41,7 @@ from rest_framework.exceptions import PermissionDenied import build.validators +import common.status_codes import InvenTree.fields import InvenTree.helpers import InvenTree.models @@ -3074,3 +3075,38 @@ def after_custom_unit_updated(sender, instance, **kwargs): from InvenTree.conversion import reload_unit_registry reload_unit_registry() + + +class DataImportSession(models.Model): + """Database model representing a data import session. + + An initial file is uploaded, and used to populate the database. + + Fields: + data_file: FileField for the data file to import + user: ForeignKey to the User who initiated the import + progress: IntegerField for the progress of the import (number of rows imported) + data_columns: JSONField for the data columns in the import file (mapped to database columns) + """ + + data_file = models.FileField( + upload_to='import', + verbose_name=_('Data File'), + help_text=_('Data file to import'), + ) + + status = models.PositiveIntegerField( + default=common.status_codes.DataImportStatusCode.INITIAL.value, + choices=common.status_codes.DataImportStatusCode.items(), + help_text=_('Import status'), + ) + + user = models.ForeignKey( + User, on_delete=models.CASCADE, blank=True, null=True, verbose_name=_('User') + ) + + progress = models.PositiveIntegerField(default=0, verbose_name=_('Progress')) + + data_columns = models.JSONField( + blank=True, null=True, verbose_name=_('Data Columns') + ) diff --git a/InvenTree/common/status_codes.py b/InvenTree/common/status_codes.py new file mode 100644 index 000000000000..f3d6867fba0e --- /dev/null +++ b/InvenTree/common/status_codes.py @@ -0,0 +1,18 @@ +"""Status codes for common model types.""" + +from django.utils.translation import gettext_lazy as _ + +from generic.states import StatusCode + + +class DataImportStatusCode(StatusCode): + """Defines a set of status codes for a DataImportSession.""" + + INITIAL = 0, _('Initial'), 'secondary' # Import session has been created + MAPPED_FIELDS = ( + 10, + _('Mapped Fields'), + 'primary', + ) # Import fields have been mapped successfully + IMPORTING = 20, _('Importing'), 'primary' # Data is being imported + COMPLETE = 30, _('Complete'), 'success' # Import has been completed From b12a0534d653e5d41c8156372aa22bf279208f27 Mon Sep 17 00:00:00 2001 From: Oliver Date: Thu, 21 Mar 2024 23:34:06 +0000 Subject: [PATCH 002/190] Add file extension validation Expose to admin interface also --- InvenTree/InvenTree/admin.py | 11 +++++++++++ InvenTree/InvenTree/helpers.py | 4 ++-- .../common/migrations/0023_dataimportsession.py | 4 +++- InvenTree/common/models.py | 14 ++++++++++++-- 4 files changed, 28 insertions(+), 5 deletions(-) diff --git a/InvenTree/InvenTree/admin.py b/InvenTree/InvenTree/admin.py index 1c79d07cc145..74b479814276 100644 --- a/InvenTree/InvenTree/admin.py +++ b/InvenTree/InvenTree/admin.py @@ -9,6 +9,8 @@ from import_export.exceptions import ImportExportError from import_export.resources import ModelResource +import common.models + class InvenTreeResource(ModelResource): """Custom subclass of the ModelResource class provided by django-import-export". @@ -109,5 +111,14 @@ def has_add_permission(self, request: HttpRequest) -> bool: return False +@admin.register(common.models.DataImportSession) +class DataImportSessionAdmin(admin.ModelAdmin): + """Admin interface for the DataImportSession model.""" + + list_display = ['id', 'data_file', 'status', 'progress', 'user'] + + list_filter = ['status'] + + admin.site.unregister(Rate) admin.site.register(Rate, CustomRateAdmin) diff --git a/InvenTree/InvenTree/helpers.py b/InvenTree/InvenTree/helpers.py index 1e30999cec07..2a4ef80820e2 100644 --- a/InvenTree/InvenTree/helpers.py +++ b/InvenTree/InvenTree/helpers.py @@ -432,8 +432,8 @@ def MakeBarcode(cls_name, object_pk: int, object_data=None, **kwargs): def GetExportFormats(): - """Return a list of allowable file formats for exporting data.""" - return ['csv', 'tsv', 'xls', 'xlsx', 'json', 'yaml'] + """Return a list of allowable file formats for importing or exporting tabular data.""" + return ['csv', 'tsv', 'xls', 'xlsx'] def DownloadFile( diff --git a/InvenTree/common/migrations/0023_dataimportsession.py b/InvenTree/common/migrations/0023_dataimportsession.py index 00d87b186eff..f555185e749d 100644 --- a/InvenTree/common/migrations/0023_dataimportsession.py +++ b/InvenTree/common/migrations/0023_dataimportsession.py @@ -3,8 +3,10 @@ from django.conf import settings from django.db import migrations, models import django.db.models.deletion +import django.core.validators import common.status_codes +import InvenTree.helpers class Migration(migrations.Migration): @@ -19,7 +21,7 @@ class Migration(migrations.Migration): name='DataImportSession', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data_file', models.FileField(help_text='Data file to import', upload_to='import', verbose_name='Data File')), + ('data_file', models.FileField(help_text='Data file to import', upload_to='import', verbose_name='Data File', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=InvenTree.helpers.GetExportFormats())])), ('status', models.PositiveIntegerField(choices=common.status_codes.DataImportStatusCode.items(), default=0, help_text='Import status')), ('progress', models.PositiveIntegerField(default=0, verbose_name='Progress')), ('data_columns', models.JSONField(blank=True, null=True, verbose_name='Data Columns')), diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index 20c6dee8abc6..d77904d938af 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -25,8 +25,13 @@ from django.contrib.contenttypes.models import ContentType from django.contrib.humanize.templatetags.humanize import naturaltime from django.core.cache import cache -from django.core.exceptions import AppRegistryNotReady, ValidationError -from django.core.validators import MaxValueValidator, MinValueValidator, URLValidator +from django.core.exceptions import ValidationError +from django.core.validators import ( + FileExtensionValidator, + MaxValueValidator, + MinValueValidator, + URLValidator, +) from django.db import models, transaction from django.db.models.signals import post_delete, post_save from django.db.utils import IntegrityError, OperationalError, ProgrammingError @@ -3093,6 +3098,11 @@ class DataImportSession(models.Model): upload_to='import', verbose_name=_('Data File'), help_text=_('Data file to import'), + validators=[ + FileExtensionValidator( + allowed_extensions=InvenTree.helpers.GetExportFormats() + ) + ], ) status = models.PositiveIntegerField( From c4ce8d644e9094680411968e5a702137692fb092 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 01:13:09 +0000 Subject: [PATCH 003/190] Switch to new 'importer' app --- InvenTree/InvenTree/admin.py | 11 --- InvenTree/InvenTree/settings.py | 1 + InvenTree/common/models.py | 48 +---------- InvenTree/importer/__init__.py | 0 InvenTree/importer/admin.py | 14 +++ InvenTree/importer/apps.py | 10 +++ .../migrations/0001_initial.py} | 19 +++-- InvenTree/importer/migrations/__init__.py | 0 InvenTree/importer/models.py | 85 +++++++++++++++++++ .../{common => importer}/status_codes.py | 0 InvenTree/importer/tests.py | 5 ++ InvenTree/importer/validators.py | 11 +++ InvenTree/importer/views.py | 5 ++ tasks.py | 2 + 14 files changed, 144 insertions(+), 67 deletions(-) create mode 100644 InvenTree/importer/__init__.py create mode 100644 InvenTree/importer/admin.py create mode 100644 InvenTree/importer/apps.py rename InvenTree/{common/migrations/0023_dataimportsession.py => importer/migrations/0001_initial.py} (51%) create mode 100644 InvenTree/importer/migrations/__init__.py create mode 100644 InvenTree/importer/models.py rename InvenTree/{common => importer}/status_codes.py (100%) create mode 100644 InvenTree/importer/tests.py create mode 100644 InvenTree/importer/validators.py create mode 100644 InvenTree/importer/views.py diff --git a/InvenTree/InvenTree/admin.py b/InvenTree/InvenTree/admin.py index 74b479814276..1c79d07cc145 100644 --- a/InvenTree/InvenTree/admin.py +++ b/InvenTree/InvenTree/admin.py @@ -9,8 +9,6 @@ from import_export.exceptions import ImportExportError from import_export.resources import ModelResource -import common.models - class InvenTreeResource(ModelResource): """Custom subclass of the ModelResource class provided by django-import-export". @@ -111,14 +109,5 @@ def has_add_permission(self, request: HttpRequest) -> bool: return False -@admin.register(common.models.DataImportSession) -class DataImportSessionAdmin(admin.ModelAdmin): - """Admin interface for the DataImportSession model.""" - - list_display = ['id', 'data_file', 'status', 'progress', 'user'] - - list_filter = ['status'] - - admin.site.unregister(Rate) admin.site.register(Rate, CustomRateAdmin) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index ff322ac10d5a..b631f3db7a01 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -193,6 +193,7 @@ 'common.apps.CommonConfig', 'company.apps.CompanyConfig', 'plugin.apps.PluginAppConfig', # Plugin app runs before all apps that depend on the isPluginRegistryLoaded function + 'importer.apps.ImporterConfig', 'label.apps.LabelConfig', 'order.apps.OrderConfig', 'part.apps.PartConfig', diff --git a/InvenTree/common/models.py b/InvenTree/common/models.py index d77904d938af..21074ebc8741 100644 --- a/InvenTree/common/models.py +++ b/InvenTree/common/models.py @@ -26,12 +26,7 @@ from django.contrib.humanize.templatetags.humanize import naturaltime from django.core.cache import cache from django.core.exceptions import ValidationError -from django.core.validators import ( - FileExtensionValidator, - MaxValueValidator, - MinValueValidator, - URLValidator, -) +from django.core.validators import MaxValueValidator, MinValueValidator, URLValidator from django.db import models, transaction from django.db.models.signals import post_delete, post_save from django.db.utils import IntegrityError, OperationalError, ProgrammingError @@ -46,7 +41,6 @@ from rest_framework.exceptions import PermissionDenied import build.validators -import common.status_codes import InvenTree.fields import InvenTree.helpers import InvenTree.models @@ -3080,43 +3074,3 @@ def after_custom_unit_updated(sender, instance, **kwargs): from InvenTree.conversion import reload_unit_registry reload_unit_registry() - - -class DataImportSession(models.Model): - """Database model representing a data import session. - - An initial file is uploaded, and used to populate the database. - - Fields: - data_file: FileField for the data file to import - user: ForeignKey to the User who initiated the import - progress: IntegerField for the progress of the import (number of rows imported) - data_columns: JSONField for the data columns in the import file (mapped to database columns) - """ - - data_file = models.FileField( - upload_to='import', - verbose_name=_('Data File'), - help_text=_('Data file to import'), - validators=[ - FileExtensionValidator( - allowed_extensions=InvenTree.helpers.GetExportFormats() - ) - ], - ) - - status = models.PositiveIntegerField( - default=common.status_codes.DataImportStatusCode.INITIAL.value, - choices=common.status_codes.DataImportStatusCode.items(), - help_text=_('Import status'), - ) - - user = models.ForeignKey( - User, on_delete=models.CASCADE, blank=True, null=True, verbose_name=_('User') - ) - - progress = models.PositiveIntegerField(default=0, verbose_name=_('Progress')) - - data_columns = models.JSONField( - blank=True, null=True, verbose_name=_('Data Columns') - ) diff --git a/InvenTree/importer/__init__.py b/InvenTree/importer/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/InvenTree/importer/admin.py b/InvenTree/importer/admin.py new file mode 100644 index 000000000000..4e78dc7e9c1f --- /dev/null +++ b/InvenTree/importer/admin.py @@ -0,0 +1,14 @@ +"""Admin site specification for the 'importer' app.""" + +from django.contrib import admin + +import importer.models + + +@admin.register(importer.models.DataImportSession) +class DataImportSessionAdmin(admin.ModelAdmin): + """Admin interface for the DataImportSession model.""" + + list_display = ['id', 'data_file', 'status', 'progress', 'user'] + + list_filter = ['status'] diff --git a/InvenTree/importer/apps.py b/InvenTree/importer/apps.py new file mode 100644 index 000000000000..4b909df3d23b --- /dev/null +++ b/InvenTree/importer/apps.py @@ -0,0 +1,10 @@ +"""AppConfig for the 'importer' app.""" + +from django.apps import AppConfig + + +class ImporterConfig(AppConfig): + """AppConfig class for the 'importer' app.""" + + default_auto_field = 'django.db.models.BigAutoField' + name = 'importer' diff --git a/InvenTree/common/migrations/0023_dataimportsession.py b/InvenTree/importer/migrations/0001_initial.py similarity index 51% rename from InvenTree/common/migrations/0023_dataimportsession.py rename to InvenTree/importer/migrations/0001_initial.py index f555185e749d..d7058dbbbd05 100644 --- a/InvenTree/common/migrations/0023_dataimportsession.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -1,30 +1,31 @@ -# Generated by Django 4.2.11 on 2024-03-21 23:12 +# Generated by Django 4.2.11 on 2024-03-22 01:09 from django.conf import settings +import django.core.validators from django.db import migrations, models import django.db.models.deletion -import django.core.validators - -import common.status_codes -import InvenTree.helpers +import importer.validators class Migration(migrations.Migration): + initial = True + dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('common', '0022_projectcode_responsible'), ] operations = [ migrations.CreateModel( name='DataImportSession', fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data_file', models.FileField(help_text='Data file to import', upload_to='import', verbose_name='Data File', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=InvenTree.helpers.GetExportFormats())])), - ('status', models.PositiveIntegerField(choices=common.status_codes.DataImportStatusCode.items(), default=0, help_text='Import status')), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['csv', 'tsv', 'xls', 'xlsx'])], verbose_name='Data File')), + ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), + ('status', models.PositiveIntegerField(choices=[(0, 'Initial'), (10, 'Mapped Fields'), (20, 'Importing'), (30, 'Complete')], default=0, help_text='Import status')), ('progress', models.PositiveIntegerField(default=0, verbose_name='Progress')), ('data_columns', models.JSONField(blank=True, null=True, verbose_name='Data Columns')), + ('field_overrides', models.JSONField(blank=True, null=True, verbose_name='Field Overrides')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], ), diff --git a/InvenTree/importer/migrations/__init__.py b/InvenTree/importer/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py new file mode 100644 index 000000000000..ce8fb4dfda13 --- /dev/null +++ b/InvenTree/importer/models.py @@ -0,0 +1,85 @@ +"""Model definitions for the 'importer' app.""" + +from django.contrib.auth.models import User +from django.core.validators import FileExtensionValidator +from django.db import models +from django.utils.translation import gettext_lazy as _ + +import importer.validators +import InvenTree.helpers +from importer.status_codes import DataImportStatusCode +from importer.validators import validate_importer_model_type + + +class DataImportSession(models.Model): + """Database model representing a data import session. + + An initial file is uploaded, and used to populate the database. + + Fields: + data_file: FileField for the data file to import + status: IntegerField for the status of the import session + user: ForeignKey to the User who initiated the import + progress: IntegerField for the progress of the import (number of rows imported) + data_columns: JSONField for the data columns in the import file (mapped to database columns) + field_overrides: JSONField for field overrides (e.g. custom field values) + """ + + @staticmethod + def supported_serializers(): + """Return a list of supported serializers which can be used for importing data.""" + import part.serializers + import stock.serializers + + return [part.serializers.PartSerializer, part.serializers.CategorySerializer] + + @staticmethod + def serializer_model_map(): + """Map supported models to their respective serializers.""" + data = {} + + for serializer in DataImportSession.supported_serializers(): + model = serializer.Meta.model + data[model.__name__.lower()] = serializer + + return data + + @staticmethod + def supported_models(): + """Return a list of database models which can be imported.""" + return list(DataImportSession.serializer_model_map().keys()) + + data_file = models.FileField( + upload_to='import', + verbose_name=_('Data File'), + help_text=_('Data file to import'), + validators=[ + FileExtensionValidator( + allowed_extensions=InvenTree.helpers.GetExportFormats() + ) + ], + ) + + model_type = models.CharField( + blank=False, max_length=100, validators=[validate_importer_model_type] + ) + + status = models.PositiveIntegerField( + default=DataImportStatusCode.INITIAL.value, + choices=DataImportStatusCode.items(), + help_text=_('Import status'), + ) + + user = models.ForeignKey( + User, on_delete=models.CASCADE, blank=True, null=True, verbose_name=_('User') + ) + + progress = models.PositiveIntegerField(default=0, verbose_name=_('Progress')) + + data_columns = models.JSONField( + blank=True, null=True, verbose_name=_('Data Columns') + ) + + field_overrides = models.JSONField( + blank=True, null=True, verbose_name=_('Field Overrides') + ) diff --git a/InvenTree/common/status_codes.py b/InvenTree/importer/status_codes.py similarity index 100% rename from InvenTree/common/status_codes.py rename to InvenTree/importer/status_codes.py diff --git a/InvenTree/importer/tests.py b/InvenTree/importer/tests.py new file mode 100644 index 000000000000..638590551cf2 --- /dev/null +++ b/InvenTree/importer/tests.py @@ -0,0 +1,5 @@ +"""Unit tests for the 'importer' app.""" + +from django.test import TestCase + +# Create your tests here. diff --git a/InvenTree/importer/validators.py b/InvenTree/importer/validators.py new file mode 100644 index 000000000000..e30bf988a9d0 --- /dev/null +++ b/InvenTree/importer/validators.py @@ -0,0 +1,11 @@ +"""Custom validation routines for the 'importer' app.""" + +from django.core.exceptions import ValidationError + + +def validate_importer_model_type(value): + """Validate that the given model type is supported for importing data.""" + from .models import DataImportSession + + if value not in DataImportSession.supported_models(): + raise ValidationError('Model type is not supported for importing data') diff --git a/InvenTree/importer/views.py b/InvenTree/importer/views.py new file mode 100644 index 000000000000..0f2e2926adcf --- /dev/null +++ b/InvenTree/importer/views.py @@ -0,0 +1,5 @@ +"""Viewsets for the 'importer' app.""" + +from django.shortcuts import render + +# Create your views here. diff --git a/tasks.py b/tasks.py index 2c2b56089b2b..0292a14f75e7 100644 --- a/tasks.py +++ b/tasks.py @@ -47,7 +47,9 @@ def apps(): 'build', 'common', 'company', + 'importer', 'label', + 'machine', 'order', 'part', 'report', From 55593c7c4a41cce011524dcb3ca23a533292b6e4 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 01:23:24 +0000 Subject: [PATCH 004/190] Refactoring to help prevent circular imports --- InvenTree/importer/models.py | 29 +++------------------ InvenTree/importer/validators.py | 43 ++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index ce8fb4dfda13..224bebf6a99f 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -8,7 +8,6 @@ import importer.validators import InvenTree.helpers from importer.status_codes import DataImportStatusCode -from importer.validators import validate_importer_model_type class DataImportSession(models.Model): @@ -25,30 +24,6 @@ class DataImportSession(models.Model): field_overrides: JSONField for field overrides (e.g. custom field values) """ - @staticmethod - def supported_serializers(): - """Return a list of supported serializers which can be used for importing data.""" - import part.serializers - import stock.serializers - - return [part.serializers.PartSerializer, part.serializers.CategorySerializer] - - @staticmethod - def serializer_model_map(): - """Map supported models to their respective serializers.""" - data = {} - - for serializer in DataImportSession.supported_serializers(): - model = serializer.Meta.model - data[model.__name__.lower()] = serializer - - return data - - @staticmethod - def supported_models(): - """Return a list of database models which can be imported.""" - return list(DataImportSession.serializer_model_map().keys()) - data_file = models.FileField( upload_to='import', verbose_name=_('Data File'), @@ -61,7 +36,9 @@ def supported_models(): ) model_type = models.CharField( - blank=False, max_length=100, validators=[validate_importer_model_type] + blank=False, + max_length=100, + validators=[importer.validators.validate_importer_model_type], ) status = models.PositiveIntegerField( diff --git a/InvenTree/importer/validators.py b/InvenTree/importer/validators.py index e30bf988a9d0..eb29a3fc2326 100644 --- a/InvenTree/importer/validators.py +++ b/InvenTree/importer/validators.py @@ -3,9 +3,42 @@ from django.core.exceptions import ValidationError -def validate_importer_model_type(value): - """Validate that the given model type is supported for importing data.""" - from .models import DataImportSession +def supported_import_serializers(): + """Return a list of supported serializers which can be used for importing data. + + TODO: Add a @decorator to serializers for auto discovery (this will allow plugins to import!) + """ + try: + import part.serializers + import stock.serializers + + return [ + part.serializers.PartSerializer, + part.serializers.CategorySerializer, + stock.serializers.LocationSerializer, + ] + except Exception: + # If the app registry is not loaded yet, return an empty list + return [] + + +def supported_models(): + """Return a map of supported models to their respective serializers.""" + data = {} + + for serializer in supported_import_serializers(): + model = serializer.Meta.model + data[model.__name__.lower()] = serializer - if value not in DataImportSession.supported_models(): - raise ValidationError('Model type is not supported for importing data') + return data + + +def allowed_importer_model_types(): + """Returns a list of allowed model type values for the DataImportSession model.""" + return supported_models().keys() + + +def validate_importer_model_type(value): + """Validate that the given model type is supported for importing.""" + if value not in allowed_importer_model_types(): + raise ValidationError(f"Unsupported model type '{value}'") From 52b530273229eafe73e45f36ca41a3574fc5b7fb Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 01:46:09 +0000 Subject: [PATCH 005/190] Add serializer registry - Use @register_importer tag for any serializer class --- InvenTree/InvenTree/helpers_model.py | 2 +- InvenTree/importer/registry.py | 47 ++++++++++++++++++++++++++++ InvenTree/importer/validators.py | 21 +++---------- InvenTree/part/serializers.py | 19 +++++++++-- 4 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 InvenTree/importer/registry.py diff --git a/InvenTree/InvenTree/helpers_model.py b/InvenTree/InvenTree/helpers_model.py index 7e87d4431487..89197f22d230 100644 --- a/InvenTree/InvenTree/helpers_model.py +++ b/InvenTree/InvenTree/helpers_model.py @@ -266,7 +266,7 @@ def render_currency( def getModelsWithMixin(mixin_class) -> list: - """Return a list of models that inherit from the given mixin class. + """Return a list of database models that inherit from the given mixin class. Args: mixin_class: The mixin class to search for diff --git a/InvenTree/importer/registry.py b/InvenTree/importer/registry.py new file mode 100644 index 000000000000..a3c8bc9ffa42 --- /dev/null +++ b/InvenTree/importer/registry.py @@ -0,0 +1,47 @@ +"""Registry for supported serializers for data import operations.""" + +import logging + +from rest_framework.serializers import Serializer + +logger = logging.getLogger('inventree') + + +class DataImportSerializerRegister: + """Registry for supported serializers for data import operations. + + To add a new serializer to the registry, add the @register_importer decorator to the serializer class. + """ + + supported_serializers: list[Serializer] = [] + + def register(self, serializer: Serializer): + """Register a new serializer with the importer registry.""" + if not isinstance(serializer, Serializer) and not issubclass( + serializer, Serializer + ): + logger.error(f'Invalid serializer type: %s', type(serializer)) + return + + logger.debug('Registering serializer class for import: %s', type(serializer)) + + if serializer not in self.supported_serializers: + self.supported_serializers.append(serializer) + + +_serializer_registry = DataImportSerializerRegister() + + +def get_supported_serializers(): + """Return a list of supported serializers which can be used for importing data.""" + return _serializer_registry.supported_serializers + + +def register_importer(): + """Decorator function to register a serializer with the importer registry.""" + + def _decorator(cls): + _serializer_registry.register(cls) + return cls + + return _decorator diff --git a/InvenTree/importer/validators.py b/InvenTree/importer/validators.py index eb29a3fc2326..f4307589bcfc 100644 --- a/InvenTree/importer/validators.py +++ b/InvenTree/importer/validators.py @@ -4,22 +4,11 @@ def supported_import_serializers(): - """Return a list of supported serializers which can be used for importing data. - - TODO: Add a @decorator to serializers for auto discovery (this will allow plugins to import!) - """ - try: - import part.serializers - import stock.serializers - - return [ - part.serializers.PartSerializer, - part.serializers.CategorySerializer, - stock.serializers.LocationSerializer, - ] - except Exception: - # If the app registry is not loaded yet, return an empty list - return [] + """Return a list of supported serializers which can be used for importing data.""" + import importer.registry + + val = importer.registry.get_supported_serializers() + return val def supported_models(): diff --git a/InvenTree/part/serializers.py b/InvenTree/part/serializers.py index f02ade9ed942..e2e9651b0cc5 100644 --- a/InvenTree/part/serializers.py +++ b/InvenTree/part/serializers.py @@ -33,6 +33,7 @@ import part.tasks import stock.models import users.models +from importer.registry import register_importer from InvenTree.status_codes import BuildStatusGroups from InvenTree.tasks import offload_task @@ -58,6 +59,7 @@ logger = logging.getLogger('inventree') +@register_importer() class CategorySerializer(InvenTree.serializers.InvenTreeModelSerializer): """Serializer for PartCategory.""" @@ -159,6 +161,7 @@ class Meta: ]) +@register_importer() class PartTestTemplateSerializer(InvenTree.serializers.InvenTreeModelSerializer): """Serializer for the PartTestTemplate class.""" @@ -258,6 +261,7 @@ def validate_image(self, value): image = InvenTree.serializers.InvenTreeAttachmentSerializerField(required=True) +@register_importer() class PartParameterTemplateSerializer(InvenTree.serializers.InvenTreeModelSerializer): """JSON serializer for the PartParameterTemplate model.""" @@ -336,6 +340,7 @@ def __init__(self, *args, **kwargs): ) +@register_importer() class PartParameterSerializer(InvenTree.serializers.InvenTreeModelSerializer): """JSON serializers for the PartParameter model.""" @@ -563,6 +568,7 @@ def validate(self, data): return data +@register_importer() class PartSerializer( InvenTree.serializers.RemoteImageMixin, InvenTree.serializers.InvenTreeTagModelSerializer, @@ -1394,6 +1400,7 @@ class Meta: ) +@register_importer() class BomItemSerializer(InvenTree.serializers.InvenTreeModelSerializer): """Serializer for BomItem object.""" @@ -1630,6 +1637,7 @@ def annotate_queryset(queryset): return queryset +@register_importer() class CategoryParameterTemplateSerializer( InvenTree.serializers.InvenTreeModelSerializer ): @@ -1722,7 +1730,10 @@ def save(self): class BomImportUploadSerializer(InvenTree.serializers.DataFileUploadSerializer): - """Serializer for uploading a file and extracting data from it.""" + """Serializer for uploading a file and extracting data from it. + + TODO: Delete this entirely once the new importer process is working + """ TARGET_MODEL = BomItem @@ -1755,6 +1766,8 @@ class BomImportExtractSerializer(InvenTree.serializers.DataFileExtractSerializer """Serializer class for exatracting BOM data from an uploaded file. The parent class DataFileExtractSerializer does most of the heavy lifting here. + + TODO: Delete this entirely once the new importer process is working """ TARGET_MODEL = BomItem @@ -1842,7 +1855,9 @@ def process_row(row): class BomImportSubmitSerializer(serializers.Serializer): """Serializer for uploading a BOM against a specified part. - A "BOM" is a set of BomItem objects which are to be validated together as a set + A "BOM" is a set of BomItem objects which are to be validated together as a set. + + TODO: Delete this entirely once the new importer process is working """ items = BomItemSerializer(many=True, required=True) From 05f8dcb3b44dbcf1ed5639f571690e89aa676a71 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 01:50:22 +0000 Subject: [PATCH 006/190] Cleanup migration file - Do not use one-time hard-coded values here --- InvenTree/importer/migrations/0001_initial.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index d7058dbbbd05..312ff0babb17 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -6,6 +6,9 @@ import django.db.models.deletion import importer.validators +from InvenTree.helpers import GetExportFormats +from importer.status_codes import DataImportStatusCode + class Migration(migrations.Migration): @@ -20,9 +23,9 @@ class Migration(migrations.Migration): name='DataImportSession', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['csv', 'tsv', 'xls', 'xlsx'])], verbose_name='Data File')), + ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=GetExportFormats())], verbose_name='Data File')), ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), - ('status', models.PositiveIntegerField(choices=[(0, 'Initial'), (10, 'Mapped Fields'), (20, 'Importing'), (30, 'Complete')], default=0, help_text='Import status')), + ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), ('progress', models.PositiveIntegerField(default=0, verbose_name='Progress')), ('data_columns', models.JSONField(blank=True, null=True, verbose_name='Data Columns')), ('field_overrides', models.JSONField(blank=True, null=True, verbose_name='Field Overrides')), From ed21dcf258030c74530d2c2feb8488940d19ffd5 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 02:15:10 +0000 Subject: [PATCH 007/190] Refactor code into registry.py --- InvenTree/importer/registry.py | 11 +++++++++++ InvenTree/importer/validators.py | 28 +++------------------------- 2 files changed, 14 insertions(+), 25 deletions(-) diff --git a/InvenTree/importer/registry.py b/InvenTree/importer/registry.py index a3c8bc9ffa42..46216c6e551c 100644 --- a/InvenTree/importer/registry.py +++ b/InvenTree/importer/registry.py @@ -37,6 +37,17 @@ def get_supported_serializers(): return _serializer_registry.supported_serializers +def supported_models(): + """Return a map of supported models to their respective serializers.""" + data = {} + + for serializer in get_supported_serializers(): + model = serializer.Meta.model + data[model.__name__.lower()] = serializer + + return data + + def register_importer(): """Decorator function to register a serializer with the importer registry.""" diff --git a/InvenTree/importer/validators.py b/InvenTree/importer/validators.py index f4307589bcfc..dac549161134 100644 --- a/InvenTree/importer/validators.py +++ b/InvenTree/importer/validators.py @@ -3,31 +3,9 @@ from django.core.exceptions import ValidationError -def supported_import_serializers(): - """Return a list of supported serializers which can be used for importing data.""" - import importer.registry - - val = importer.registry.get_supported_serializers() - return val - - -def supported_models(): - """Return a map of supported models to their respective serializers.""" - data = {} - - for serializer in supported_import_serializers(): - model = serializer.Meta.model - data[model.__name__.lower()] = serializer - - return data - - -def allowed_importer_model_types(): - """Returns a list of allowed model type values for the DataImportSession model.""" - return supported_models().keys() - - def validate_importer_model_type(value): """Validate that the given model type is supported for importing.""" - if value not in allowed_importer_model_types(): + from importer.registry import supported_models + + if value not in supported_models().keys(): raise ValidationError(f"Unsupported model type '{value}'") From 46371c4ae80b20a5e3d5a6945335780e31d348f7 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 03:25:46 +0000 Subject: [PATCH 008/190] Add validation for the uploaded file - Must be importable by tablib --- InvenTree/importer/migrations/0001_initial.py | 2 +- InvenTree/importer/models.py | 3 +- InvenTree/importer/operations.py | 66 +++++++++++++++++++ InvenTree/importer/validators.py | 27 ++++++++ 4 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 InvenTree/importer/operations.py diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index 312ff0babb17..aad0eaa815b0 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -23,7 +23,7 @@ class Migration(migrations.Migration): name='DataImportSession', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=GetExportFormats())], verbose_name='Data File')), + ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=GetExportFormats()), importer.validators.validate_data_file], verbose_name='Data File')), ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), ('progress', models.PositiveIntegerField(default=0, verbose_name='Progress')), diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 224bebf6a99f..115bbffeec55 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -31,7 +31,8 @@ class DataImportSession(models.Model): validators=[ FileExtensionValidator( allowed_extensions=InvenTree.helpers.GetExportFormats() - ) + ), + importer.validators.validate_data_file, ], ) diff --git a/InvenTree/importer/operations.py b/InvenTree/importer/operations.py new file mode 100644 index 000000000000..aca2085e2582 --- /dev/null +++ b/InvenTree/importer/operations.py @@ -0,0 +1,66 @@ +"""Data import operational functions.""" + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +import tablib + + +def load_data_file(data_file, format): + """Load data file into a tablib dataset. + + Arguments: + data_file: File object containing data to import (should be already opened!) + format: Format specifier for the data file + """ + if format and format.startswith('.'): + format = format[1:] + + data_file.open('r') + data_file.seek(0) + + try: + data = data_file.read() + except (IOError, FileNotFoundError): + raise ValidationError(_('Failed to open data file')) + + if format not in ['xls', 'xlsx']: + data = data.decode() + + try: + data = tablib.Dataset().load(data, headers=True, format=format) + except tablib.core.UnsupportedFormat: + raise ValidationError(_('Unsupported data file format')) + except tablib.core.InvalidDimensions: + raise ValidationError(_('Invalid data file dimensions')) + + # TODO: raise exceptions! + + return data + + +def extract_column_names(data_file) -> list: + """Extract column names from a data file. + + Uses the tablib library to extract column names from a data file. + + Args: + data_file: File object containing data to import + + Returns: + List of column names extracted from the file + + Raises: + ValidationError: If the data file is not in a valid format + """ + try: + with open(data_file, 'r') as fh: + data = tablib.Dataset(fh, headers=True) + except (IOError, FileNotFoundError): + raise ValidationError('Failed to open data file') + except tablib.core.UnsupportedFormat: + raise ValidationError('Unsupported data file format') + except tablib.core.InvalidDimensions: + raise ValidationError('Invalid data file dimensions') + + return data.headers diff --git a/InvenTree/importer/validators.py b/InvenTree/importer/validators.py index dac549161134..f97387929e23 100644 --- a/InvenTree/importer/validators.py +++ b/InvenTree/importer/validators.py @@ -1,6 +1,33 @@ """Custom validation routines for the 'importer' app.""" +import os + from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +# Define maximum limits for imported file data +IMPORTER_MAX_FILE_SIZE = 32 * 1024 * 1042 +IMPORTER_MAX_ROWS = 1000 +IMPORTER_MAX_COLS = 1000 + + +def validate_data_file(data_file): + """Validate the provided data file.""" + import importer.operations + + filesize = data_file.size + filetype = os.path.splitext(data_file.name)[1] + + if filesize > IMPORTER_MAX_FILE_SIZE: + raise ValidationError(_('Data file exceeds maximum size limit')) + + dataset = importer.operations.load_data_file(data_file.file, format=filetype) + + if len(dataset.headers) > IMPORTER_MAX_COLS: + raise ValidationError(_('Data file contains too many columns')) + + if len(dataset) > IMPORTER_MAX_ROWS: + raise ValidationError(_('Data file contains too many rows')) def validate_importer_model_type(value): From b0fc755b19da704109f723ea1e820f91be14f593 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 03:29:15 +0000 Subject: [PATCH 009/190] Refactoring --- InvenTree/importer/operations.py | 17 ++++++++++------- InvenTree/importer/validators.py | 3 +-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/InvenTree/importer/operations.py b/InvenTree/importer/operations.py index aca2085e2582..04003d1dca62 100644 --- a/InvenTree/importer/operations.py +++ b/InvenTree/importer/operations.py @@ -6,21 +6,26 @@ import tablib -def load_data_file(data_file, format): +def load_data_file(data_file, format=None): """Load data file into a tablib dataset. Arguments: - data_file: File object containing data to import (should be already opened!) + data_file: django file object containing data to import (should be already opened!) format: Format specifier for the data file """ + # Introspect the file format based on the provided file + if not format: + format = data_file.name.split('.')[-1] + if format and format.startswith('.'): format = format[1:] - data_file.open('r') - data_file.seek(0) + file_object = data_file.file + file_object.open('r') + file_object.seek(0) try: - data = data_file.read() + data = file_object.read() except (IOError, FileNotFoundError): raise ValidationError(_('Failed to open data file')) @@ -34,8 +39,6 @@ def load_data_file(data_file, format): except tablib.core.InvalidDimensions: raise ValidationError(_('Invalid data file dimensions')) - # TODO: raise exceptions! - return data diff --git a/InvenTree/importer/validators.py b/InvenTree/importer/validators.py index f97387929e23..1048498f73ae 100644 --- a/InvenTree/importer/validators.py +++ b/InvenTree/importer/validators.py @@ -16,12 +16,11 @@ def validate_data_file(data_file): import importer.operations filesize = data_file.size - filetype = os.path.splitext(data_file.name)[1] if filesize > IMPORTER_MAX_FILE_SIZE: raise ValidationError(_('Data file exceeds maximum size limit')) - dataset = importer.operations.load_data_file(data_file.file, format=filetype) + dataset = importer.operations.load_data_file(data_file) if len(dataset.headers) > IMPORTER_MAX_COLS: raise ValidationError(_('Data file contains too many columns')) From fa7d15fc14570508b5504510f2ba4681307eaa7c Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 03:31:48 +0000 Subject: [PATCH 010/190] Adds property to retrieve matching serializer class --- InvenTree/importer/models.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 115bbffeec55..7c9e07c51f3c 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -61,3 +61,10 @@ class DataImportSession(models.Model): field_overrides = models.JSONField( blank=True, null=True, verbose_name=_('Field Overrides') ) + + @property + def serializer(self): + """Return the serializer class for this model.""" + from importer.registry import supported_models + + return supported_models().get(self.model_type, None) From f0eba71e454490b27574da09f28b70d8a41a93a3 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 03:47:59 +0000 Subject: [PATCH 011/190] Update helper functions --- InvenTree/importer/models.py | 12 +++++++++++- InvenTree/importer/operations.py | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 7c9e07c51f3c..c7cd38e48628 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -64,7 +64,17 @@ class DataImportSession(models.Model): @property def serializer(self): - """Return the serializer class for this model.""" + """Return the serializer class for this importer.""" from importer.registry import supported_models return supported_models().get(self.model_type, None) + + def serializer_fields(self, required=None, read_only=False): + """Return the writeable serializers fields for this importer. + + Arguments: + required: If True, only return required fields + """ + from importer.operations import get_fields + + return get_fields(self.serializer, required=required, read_only=read_only) diff --git a/InvenTree/importer/operations.py b/InvenTree/importer/operations.py index 04003d1dca62..6c884f8d1bb2 100644 --- a/InvenTree/importer/operations.py +++ b/InvenTree/importer/operations.py @@ -67,3 +67,28 @@ def extract_column_names(data_file) -> list: raise ValidationError('Invalid data file dimensions') return data.headers + + +def get_fields(serializer_class, read_only=None, required=None): + """Extract the field names from a serializer class. + + - Returns a dict of (writeable) field names. + - Read-Only fields are explicitly ignored + """ + if not serializer_class: + return {} + + serializer = serializer_class() + + fields = {} + + for field_name, field in serializer.fields.items(): + if read_only is not None and getattr(field, 'read_only', None) != read_only: + continue + + if required is not None and getattr(field, 'required', None) != required: + continue + + fields[field_name] = field + + return fields From fec8c1f8b7793218309957248b09dc7a079450eb Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 04:02:59 +0000 Subject: [PATCH 012/190] Add hook to auto-assign columns on initial creation --- InvenTree/importer/migrations/0001_initial.py | 2 +- InvenTree/importer/models.py | 27 +++++++++++++++++-- InvenTree/importer/operations.py | 11 +------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index aad0eaa815b0..e40177a72634 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -28,7 +28,7 @@ class Migration(migrations.Migration): ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), ('progress', models.PositiveIntegerField(default=0, verbose_name='Progress')), ('data_columns', models.JSONField(blank=True, null=True, verbose_name='Data Columns')), - ('field_overrides', models.JSONField(blank=True, null=True, verbose_name='Field Overrides')), + ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], ), diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index c7cd38e48628..16c712c47028 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -24,6 +24,15 @@ class DataImportSession(models.Model): field_overrides: JSONField for field overrides (e.g. custom field values) """ + def save(self, *args, **kwargs): + """Save the DataImportSession object.""" + if self.pk is None: + # New object - run initial setup + self.progress = 0 + self.auto_assign_columns() + + super().save(*args, **kwargs) + data_file = models.FileField( upload_to='import', verbose_name=_('Data File'), @@ -58,8 +67,8 @@ class DataImportSession(models.Model): blank=True, null=True, verbose_name=_('Data Columns') ) - field_overrides = models.JSONField( - blank=True, null=True, verbose_name=_('Field Overrides') + field_defaults = models.JSONField( + blank=True, null=True, verbose_name=_('Field Defaults') ) @property @@ -78,3 +87,17 @@ def serializer_fields(self, required=None, read_only=False): from importer.operations import get_fields return get_fields(self.serializer, required=required, read_only=read_only) + + def auto_assign_columns(self): + """Automatically assign columns based on the serializer fields.""" + from importer.operations import extract_column_names + + available_columns = extract_column_names(self.data_file) + serializer_fields = self.serializer_fields() + + # Create a mapping of column names to serializer fields + column_map = {} + + # TODO... implement auto mapping + + self.data_columns = column_map diff --git a/InvenTree/importer/operations.py b/InvenTree/importer/operations.py index 6c884f8d1bb2..8af0878a0193 100644 --- a/InvenTree/importer/operations.py +++ b/InvenTree/importer/operations.py @@ -56,16 +56,7 @@ def extract_column_names(data_file) -> list: Raises: ValidationError: If the data file is not in a valid format """ - try: - with open(data_file, 'r') as fh: - data = tablib.Dataset(fh, headers=True) - except (IOError, FileNotFoundError): - raise ValidationError('Failed to open data file') - except tablib.core.UnsupportedFormat: - raise ValidationError('Unsupported data file format') - except tablib.core.InvalidDimensions: - raise ValidationError('Invalid data file dimensions') - + data = load_data_file(data_file) return data.headers From 7264c89f354ab5923e09600ee26aa2e5a85895b0 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 04:05:11 +0000 Subject: [PATCH 013/190] Rename field --- InvenTree/importer/migrations/0001_initial.py | 2 +- InvenTree/importer/models.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index e40177a72634..3261e85cf5d8 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -27,8 +27,8 @@ class Migration(migrations.Migration): ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), ('progress', models.PositiveIntegerField(default=0, verbose_name='Progress')), - ('data_columns', models.JSONField(blank=True, null=True, verbose_name='Data Columns')), ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults')), + ('field_mapping', models.JSONField(blank=True, null=True, verbose_name='Field Mapping')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], ), diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 16c712c47028..e76ef2cd0315 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -63,14 +63,14 @@ def save(self, *args, **kwargs): progress = models.PositiveIntegerField(default=0, verbose_name=_('Progress')) - data_columns = models.JSONField( - blank=True, null=True, verbose_name=_('Data Columns') - ) - field_defaults = models.JSONField( blank=True, null=True, verbose_name=_('Field Defaults') ) + field_mapping = models.JSONField( + blank=True, null=True, verbose_name=_('Field Mapping') + ) + @property def serializer(self): """Return the serializer class for this importer.""" From 97f91cd0969d5b41e36df0299cb4935fdea7289d Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 04:08:58 +0000 Subject: [PATCH 014/190] Enforce initial status value --- InvenTree/importer/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index e76ef2cd0315..887869c98425 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -28,6 +28,7 @@ def save(self, *args, **kwargs): """Save the DataImportSession object.""" if self.pk is None: # New object - run initial setup + self.status = DataImportStatusCode.INITIAL.value self.progress = 0 self.auto_assign_columns() From db54d301faf5389d1b656809515d7a9b5a820290 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 22 Mar 2024 04:13:11 +0000 Subject: [PATCH 015/190] Add model for individual rows in the data import --- InvenTree/importer/admin.py | 7 +++++++ InvenTree/importer/models.py | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/InvenTree/importer/admin.py b/InvenTree/importer/admin.py index 4e78dc7e9c1f..54af22596bcd 100644 --- a/InvenTree/importer/admin.py +++ b/InvenTree/importer/admin.py @@ -12,3 +12,10 @@ class DataImportSessionAdmin(admin.ModelAdmin): list_display = ['id', 'data_file', 'status', 'progress', 'user'] list_filter = ['status'] + + +@admin.register(importer.models.DataImportRow) +class DataImportRowAdmin(admin.ModelAdmin): + """Admin interface for the DataImportRow model.""" + + list_display = ['id', 'session', 'row_index'] diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 887869c98425..fd436bcb1a07 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -102,3 +102,29 @@ def auto_assign_columns(self): # TODO... implement auto mapping self.data_columns = column_map + + +class DataImportRow(models.Model): + """Database model representing a single row in a data import session. + + Each row corresponds to a single row in the import file, and is used to populate the database. + + Fields: + session: ForeignKey to the parent DataImportSession object + data: JSONField for the data in this row + status: IntegerField for the status of the row import + """ + + session = models.ForeignKey( + DataImportSession, on_delete=models.CASCADE, verbose_name=_('Import Session') + ) + + row_index = models.PositiveIntegerField(default=0, verbose_name=_('Row Index')) + + row_data = models.JSONField( + blank=True, null=True, verbose_name=_('Original row data') + ) + + data = models.JSONField(blank=True, null=True, verbose_name=_('Data')) + + errors = models.JSONField(blank=True, null=True, verbose_name=_('Errors')) From 5b251dce191f4b0c1a2db866928af9ee4484fb1c Mon Sep 17 00:00:00 2001 From: Oliver Date: Sun, 24 Mar 2024 22:03:28 +0000 Subject: [PATCH 016/190] Add DataImportRow model --- InvenTree/importer/migrations/0001_initial.py | 11 ++++++++ InvenTree/importer/models.py | 28 +++++++++++++++---- InvenTree/importer/operations.py | 15 ++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index 3261e85cf5d8..0f3a5afa13a4 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -32,4 +32,15 @@ class Migration(migrations.Migration): ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], ), + migrations.CreateModel( + name='DataImportRow', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('row_index', models.PositiveIntegerField(default=0, verbose_name='Row Index')), + ('row_data', models.JSONField(blank=True, null=True, verbose_name='Original row data')), + ('data', models.JSONField(blank=True, null=True, verbose_name='Data')), + ('errors', models.JSONField(blank=True, null=True, verbose_name='Errors')), + ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rows', to='importer.dataimportsession', verbose_name='Import Session')), + ], + ), ] diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index fd436bcb1a07..1050d0545eba 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -2,7 +2,7 @@ from django.contrib.auth.models import User from django.core.validators import FileExtensionValidator -from django.db import models +from django.db import models, transaction from django.utils.translation import gettext_lazy as _ import importer.validators @@ -26,14 +26,16 @@ class DataImportSession(models.Model): def save(self, *args, **kwargs): """Save the DataImportSession object.""" - if self.pk is None: + initial = self.pk is None + super().save(*args, **kwargs) + + if initial: # New object - run initial setup self.status = DataImportStatusCode.INITIAL.value self.progress = 0 + self.create_rows() self.auto_assign_columns() - super().save(*args, **kwargs) - data_file = models.FileField( upload_to='import', verbose_name=_('Data File'), @@ -101,7 +103,18 @@ def auto_assign_columns(self): # TODO... implement auto mapping - self.data_columns = column_map + self.field_mapping = column_map + + @transaction.atomic + def create_rows(self): + """Generate DataImportRow objects for each row in the import file.""" + from importer.operations import extract_rows + + # Remove any existing rows + self.rows.all().delete() + + for idx, row in enumerate(extract_rows(self.data_file)): + DataImportRow.objects.create(session=self, data=row, row_index=idx) class DataImportRow(models.Model): @@ -116,7 +129,10 @@ class DataImportRow(models.Model): """ session = models.ForeignKey( - DataImportSession, on_delete=models.CASCADE, verbose_name=_('Import Session') + DataImportSession, + on_delete=models.CASCADE, + verbose_name=_('Import Session'), + related_name='rows', ) row_index = models.PositiveIntegerField(default=0, verbose_name=_('Row Index')) diff --git a/InvenTree/importer/operations.py b/InvenTree/importer/operations.py index 8af0878a0193..30800b7e28ed 100644 --- a/InvenTree/importer/operations.py +++ b/InvenTree/importer/operations.py @@ -60,6 +60,21 @@ def extract_column_names(data_file) -> list: return data.headers +def extract_rows(data_file) -> list: + """Extract rows from the data file. + + Each returned row is a dictionary of column_name: value pairs. + """ + data = load_data_file(data_file) + + rows = [] + + for row in data: + rows.append(row) + + return rows + + def get_fields(serializer_class, read_only=None, required=None): """Extract the field names from a serializer class. From 58a1c72c808619bfc1f0178ed20dca3f83ff0812 Mon Sep 17 00:00:00 2001 From: Oliver Date: Sun, 24 Mar 2024 22:06:30 +0000 Subject: [PATCH 017/190] Extract data rows as dict --- InvenTree/importer/operations.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/InvenTree/importer/operations.py b/InvenTree/importer/operations.py index 30800b7e28ed..fdb8c194884f 100644 --- a/InvenTree/importer/operations.py +++ b/InvenTree/importer/operations.py @@ -67,10 +67,12 @@ def extract_rows(data_file) -> list: """ data = load_data_file(data_file) + headers = data.headers + rows = [] for row in data: - rows.append(row) + rows.append(dict(zip(headers, row))) return rows From d4e4611f04bd02a45d8bbca49674f252b5179500 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 03:34:34 +0000 Subject: [PATCH 018/190] Update fields - Remove "progress" field (will be calculated) - Added "timestamp" field - Added "complete" field to DataImportRow --- InvenTree/importer/admin.py | 2 +- InvenTree/importer/migrations/0001_initial.py | 3 ++- InvenTree/importer/models.py | 9 ++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/InvenTree/importer/admin.py b/InvenTree/importer/admin.py index 54af22596bcd..2f1507a63e9c 100644 --- a/InvenTree/importer/admin.py +++ b/InvenTree/importer/admin.py @@ -9,7 +9,7 @@ class DataImportSessionAdmin(admin.ModelAdmin): """Admin interface for the DataImportSession model.""" - list_display = ['id', 'data_file', 'status', 'progress', 'user'] + list_display = ['id', 'data_file', 'status', 'user'] list_filter = ['status'] diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index 0f3a5afa13a4..f0c3bdc7b2d1 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -23,10 +23,10 @@ class Migration(migrations.Migration): name='DataImportSession', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(auto_now_add=True, verbose_name='Timestamp')), ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=GetExportFormats()), importer.validators.validate_data_file], verbose_name='Data File')), ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), - ('progress', models.PositiveIntegerField(default=0, verbose_name='Progress')), ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults')), ('field_mapping', models.JSONField(blank=True, null=True, verbose_name='Field Mapping')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), @@ -41,6 +41,7 @@ class Migration(migrations.Migration): ('data', models.JSONField(blank=True, null=True, verbose_name='Data')), ('errors', models.JSONField(blank=True, null=True, verbose_name='Errors')), ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rows', to='importer.dataimportsession', verbose_name='Import Session')), + ('complete', models.BooleanField(default=False, verbose_name='Complete')), ], ), ] diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 1050d0545eba..9949427fc38c 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -16,10 +16,10 @@ class DataImportSession(models.Model): An initial file is uploaded, and used to populate the database. Fields: + timestamp: Timestamp for the import session data_file: FileField for the data file to import status: IntegerField for the status of the import session user: ForeignKey to the User who initiated the import - progress: IntegerField for the progress of the import (number of rows imported) data_columns: JSONField for the data columns in the import file (mapped to database columns) field_overrides: JSONField for field overrides (e.g. custom field values) """ @@ -35,6 +35,9 @@ def save(self, *args, **kwargs): self.progress = 0 self.create_rows() self.auto_assign_columns() + self.save() + + timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_('Timestamp')) data_file = models.FileField( upload_to='import', @@ -64,8 +67,6 @@ def save(self, *args, **kwargs): User, on_delete=models.CASCADE, blank=True, null=True, verbose_name=_('User') ) - progress = models.PositiveIntegerField(default=0, verbose_name=_('Progress')) - field_defaults = models.JSONField( blank=True, null=True, verbose_name=_('Field Defaults') ) @@ -144,3 +145,5 @@ class DataImportRow(models.Model): data = models.JSONField(blank=True, null=True, verbose_name=_('Data')) errors = models.JSONField(blank=True, null=True, verbose_name=_('Errors')) + + complete = models.BooleanField(default=False, verbose_name=_('Complete')) From 35dd0e9b137355a7b3f2bf14193b9a2a0e243d72 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 03:39:28 +0000 Subject: [PATCH 019/190] Auto-map column names - Provide "sensible" default values --- InvenTree/importer/models.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 9949427fc38c..efda6a681cfe 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -102,7 +102,16 @@ def auto_assign_columns(self): # Create a mapping of column names to serializer fields column_map = {} - # TODO... implement auto mapping + for name, field in serializer_fields.items(): + column = None + label = getattr(field, 'label', name) + + if name in available_columns: + column = name + elif label in available_columns: + column = label + + column_map[name] = column self.field_mapping = column_map From 002b77c67da0c7dd32dacdd9edc2c4a20c6c87bd Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 03:54:56 +0000 Subject: [PATCH 020/190] Add API endpoint for DataImportSession --- InvenTree/InvenTree/urls.py | 18 ++++++------ InvenTree/importer/api.py | 24 ++++++++++++++++ InvenTree/importer/models.py | 16 +++++++++++ InvenTree/importer/operations.py | 5 +++- InvenTree/importer/serializers.py | 48 +++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 InvenTree/importer/api.py create mode 100644 InvenTree/importer/serializers.py diff --git a/InvenTree/InvenTree/urls.py b/InvenTree/InvenTree/urls.py index 6f007a084a6e..c43bd61efbc9 100644 --- a/InvenTree/InvenTree/urls.py +++ b/InvenTree/InvenTree/urls.py @@ -21,6 +21,7 @@ import build.api import common.api import company.api +import importer.api import label.api import machine.api import order.api @@ -74,19 +75,20 @@ apipatterns = [ # Global search - path('search/', APISearchView.as_view(), name='api-search'), - path('settings/', include(common.api.settings_api_urls)), - path('part/', include(part.api.part_api_urls)), + path('admin/', include(common.api.admin_api_urls)), path('bom/', include(part.api.bom_api_urls)), - path('company/', include(company.api.company_api_urls)), - path('stock/', include(stock.api.stock_api_urls)), path('build/', include(build.api.build_api_urls)), - path('order/', include(order.api.order_api_urls)), + path('company/', include(company.api.company_api_urls)), + path('importer/', include(importer.api.importer_api_urls)), path('label/', include(label.api.label_api_urls)), - path('report/', include(report.api.report_api_urls)), path('machine/', include(machine.api.machine_api_urls)), + path('order/', include(order.api.order_api_urls)), + path('part/', include(part.api.part_api_urls)), + path('report/', include(report.api.report_api_urls)), + path('search/', APISearchView.as_view(), name='api-search'), + path('settings/', include(common.api.settings_api_urls)), + path('stock/', include(stock.api.stock_api_urls)), path('user/', include(users.api.user_urls)), - path('admin/', include(common.api.admin_api_urls)), path('web/', include(web_api_urls)), # Plugin endpoints path('', include(plugin.api.plugin_api_urls)), diff --git a/InvenTree/importer/api.py b/InvenTree/importer/api.py new file mode 100644 index 000000000000..f238503680d0 --- /dev/null +++ b/InvenTree/importer/api.py @@ -0,0 +1,24 @@ +"""API endpoints for the importer app.""" + +from django.urls import include, path + +import importer.models +import importer.serializers +from InvenTree.mixins import ListCreateAPI + + +class DataImportSessionList(ListCreateAPI): + """API endpoint for accessing a list of DataImportSession objects.""" + + queryset = importer.models.DataImportSession.objects.all() + serializer_class = importer.serializers.DataImportSessionSerializer + + +importer_api_urls = [ + path( + 'session/', + include([ + path('', DataImportSessionList.as_view(), name='importer-session-list') + ]), + ) +] diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index efda6a681cfe..230d16c055c3 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -27,6 +27,9 @@ class DataImportSession(models.Model): def save(self, *args, **kwargs): """Save the DataImportSession object.""" initial = self.pk is None + + self.clean() + super().save(*args, **kwargs) if initial: @@ -126,6 +129,19 @@ def create_rows(self): for idx, row in enumerate(extract_rows(self.data_file)): DataImportRow.objects.create(session=self, data=row, row_index=idx) + @property + def row_count(self): + """Return the number of rows in the import session.""" + return self.rows.count() + + @property + def completed_row_count(self): + """Return the number of completed rows for this session.""" + if self.row_count == 0: + return 0 + + return self.rows.filter(complete=True).count() / self.row_count * 100 + class DataImportRow(models.Model): """Database model representing a single row in a data import session. diff --git a/InvenTree/importer/operations.py b/InvenTree/importer/operations.py index fdb8c194884f..0f53d182b06f 100644 --- a/InvenTree/importer/operations.py +++ b/InvenTree/importer/operations.py @@ -21,7 +21,10 @@ def load_data_file(data_file, format=None): format = format[1:] file_object = data_file.file - file_object.open('r') + + if hasattr(file_object, 'open'): + file_object.open('r') + file_object.seek(0) try: diff --git a/InvenTree/importer/serializers.py b/InvenTree/importer/serializers.py new file mode 100644 index 000000000000..40d7c794e7df --- /dev/null +++ b/InvenTree/importer/serializers.py @@ -0,0 +1,48 @@ +"""API serializers for the importer app.""" + +from django.utils.translation import gettext_lazy as _ + +from rest_framework import serializers + +import importer.models +from InvenTree.serializers import InvenTreeModelSerializer + + +class DataImportSessionSerializer(InvenTreeModelSerializer): + """Serializer for the DataImportSession model.""" + + class Meta: + """Meta class options for the serializer.""" + + model = importer.models.DataImportSession + fields = [ + 'pk', + 'timestamp', + 'data_file', + 'model_type', + 'status', + 'user', + 'field_defaults', + 'field_mapping', + 'row_count', + 'completed_row_count', + ] + read_only_fields = ['pk', 'user', 'status'] + + row_count = serializers.IntegerField(read_only=True) + completed_row_count = serializers.IntegerField(read_only=True) + + def create(self, validated_data): + """Override create method for this serializer. + + Attach user information based on provided session data. + """ + session = super().create(validated_data) + + request = self.context.get('request', None) + + if request: + session.user = request.user + session.save() + + return session From 7e70f76f42e5c0ea509f188d8f35a802091029da Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 04:16:12 +0000 Subject: [PATCH 021/190] Offload data import operation - For large data files this may take a significant amount of time - Offload it to the background worker process --- InvenTree/importer/migrations/0001_initial.py | 1 - InvenTree/importer/models.py | 14 ++--- InvenTree/importer/tasks.py | 58 +++++++++++++++++++ InvenTree/importer/validators.py | 3 + 4 files changed, 66 insertions(+), 10 deletions(-) create mode 100644 InvenTree/importer/tasks.py diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index f0c3bdc7b2d1..001f2fd8838e 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -39,7 +39,6 @@ class Migration(migrations.Migration): ('row_index', models.PositiveIntegerField(default=0, verbose_name='Row Index')), ('row_data', models.JSONField(blank=True, null=True, verbose_name='Original row data')), ('data', models.JSONField(blank=True, null=True, verbose_name='Data')), - ('errors', models.JSONField(blank=True, null=True, verbose_name='Errors')), ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rows', to='importer.dataimportsession', verbose_name='Import Session')), ('complete', models.BooleanField(default=False, verbose_name='Complete')), ], diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 230d16c055c3..4985823f6f2e 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -36,10 +36,11 @@ def save(self, *args, **kwargs): # New object - run initial setup self.status = DataImportStatusCode.INITIAL.value self.progress = 0 - self.create_rows() self.auto_assign_columns() self.save() + self.create_rows() + timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_('Timestamp')) data_file = models.FileField( @@ -121,13 +122,10 @@ def auto_assign_columns(self): @transaction.atomic def create_rows(self): """Generate DataImportRow objects for each row in the import file.""" - from importer.operations import extract_rows + from importer.tasks import load_data + from InvenTree.tasks import offload_task - # Remove any existing rows - self.rows.all().delete() - - for idx, row in enumerate(extract_rows(self.data_file)): - DataImportRow.objects.create(session=self, data=row, row_index=idx) + offload_task(load_data, self.pk) @property def row_count(self): @@ -169,6 +167,4 @@ class DataImportRow(models.Model): data = models.JSONField(blank=True, null=True, verbose_name=_('Data')) - errors = models.JSONField(blank=True, null=True, verbose_name=_('Errors')) - complete = models.BooleanField(default=False, verbose_name=_('Complete')) diff --git a/InvenTree/importer/tasks.py b/InvenTree/importer/tasks.py new file mode 100644 index 000000000000..62cd64250fa9 --- /dev/null +++ b/InvenTree/importer/tasks.py @@ -0,0 +1,58 @@ +"""Task definitions for the 'importer' app.""" + +import logging + +from django.db import transaction + +logger = logging.getLogger('inventree') + + +@transaction.atomic +def load_data(session_id: int): + """Load data from the provided file. + + Attempt to load data from the provided file, and potentially handle any errors. + """ + import importer.models + import importer.operations + import importer.status_codes + + try: + session = importer.models.DataImportSession.objects.get(pk=session_id) + logger.info("Loading data from session ID '%s'", session_id) + except (ValueError, importer.models.DataImportSession.DoesNotExist): + logger.error("Data import session with ID '%s' does not exist", session_id) + return + + # Clear any existing data rows + session.rows.all().delete() + + df = importer.operations.load_data_file(session.data_file) + + if df is None: + # TODO: Log an error message against the import session + logger.error('Failed to load data file') + return + + headers = df.headers + + # TODO: Mark the session as "importing" + # session.status = importer.status_codes.DataImportStatusCode.IMPORTING.value + # session.save() + + row_objects = [] + + # Iterate through each "row" in the data file, and create a new DataImportRow object + for idx, row in enumerate(df): + row_data = dict(zip(headers, row)) + + row_objects.append( + importer.models.DataImportRow( + session=session, row_data=row_data, row_index=idx + ) + ) + + # Finally, create the DataImportRow objects + importer.models.DataImportRow.objects.bulk_create(row_objects) + + # TODO: Mark the import task as "completed" diff --git a/InvenTree/importer/validators.py b/InvenTree/importer/validators.py index 1048498f73ae..4f6ab8cb5814 100644 --- a/InvenTree/importer/validators.py +++ b/InvenTree/importer/validators.py @@ -22,6 +22,9 @@ def validate_data_file(data_file): dataset = importer.operations.load_data_file(data_file) + if not dataset.headers or len(dataset.headers) == 0: + raise ValidationError(_('Data file contains no headers')) + if len(dataset.headers) > IMPORTER_MAX_COLS: raise ValidationError(_('Data file contains too many columns')) From 40259011f3df95bcc93160b171513f92d8dbe3f0 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 04:31:06 +0000 Subject: [PATCH 022/190] Refactor data import code --- InvenTree/importer/models.py | 54 ++++++++++++++++++++++++++---- InvenTree/importer/status_codes.py | 9 ++--- InvenTree/importer/tasks.py | 36 ++------------------ 3 files changed, 53 insertions(+), 46 deletions(-) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 4985823f6f2e..4de216bb6ecc 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -5,6 +5,7 @@ from django.db import models, transaction from django.utils.translation import gettext_lazy as _ +import importer.operations import importer.validators import InvenTree.helpers from importer.status_codes import DataImportStatusCode @@ -39,7 +40,7 @@ def save(self, *args, **kwargs): self.auto_assign_columns() self.save() - self.create_rows() + self.trigger_data_import() timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_('Timestamp')) @@ -119,13 +120,54 @@ def auto_assign_columns(self): self.field_mapping = column_map - @transaction.atomic - def create_rows(self): - """Generate DataImportRow objects for each row in the import file.""" - from importer.tasks import load_data + def trigger_data_import(self): + """Trigger the data import process for this session. + + Offloads the task to the background worker process. + """ + from importer.tasks import import_data from InvenTree.tasks import offload_task - offload_task(load_data, self.pk) + offload_task(import_data, self.pk) + + def import_data(self): + """Perform the data import process for this session.""" + # TODO: Clear any existing error messages + + # Clear any existing data rows + self.rows.all().delete() + + df = importer.operations.load_data_file(self.data_file) + + if df is None: + # TODO: Log an error message against the import session + logger.error('Failed to load data file') + return + + headers = df.headers + + row_objects = [] + + # Mark the import task status as "IMPORTING" + self.status = DataImportStatusCode.IMPORTING.value + self.save() + + # Iterate through each "row" in the data file, and create a new DataImportRow object + for idx, row in enumerate(df): + row_data = dict(zip(headers, row)) + + row_objects.append( + importer.models.DataImportRow( + session=self, row_data=row_data, row_index=idx + ) + ) + + # Finally, create the DataImportRow objects + importer.models.DataImportRow.objects.bulk_create(row_objects) + + # Mark the import task as "PROCESSING" + self.status = DataImportStatusCode.PROCESSING.value + self.save() @property def row_count(self): diff --git a/InvenTree/importer/status_codes.py b/InvenTree/importer/status_codes.py index f3d6867fba0e..840267e0ba9c 100644 --- a/InvenTree/importer/status_codes.py +++ b/InvenTree/importer/status_codes.py @@ -9,10 +9,7 @@ class DataImportStatusCode(StatusCode): """Defines a set of status codes for a DataImportSession.""" INITIAL = 0, _('Initial'), 'secondary' # Import session has been created - MAPPED_FIELDS = ( - 10, - _('Mapped Fields'), - 'primary', - ) # Import fields have been mapped successfully + MAPPING = 10, _('Mapping'), 'primary' # Import fields are being mapped IMPORTING = 20, _('Importing'), 'primary' # Data is being imported - COMPLETE = 30, _('Complete'), 'success' # Import has been completed + PROCESSING = 30, _('Processing'), 'primary' # Data is being processed by the user + COMPLETE = 40, _('Complete'), 'success' # Import has been completed diff --git a/InvenTree/importer/tasks.py b/InvenTree/importer/tasks.py index 62cd64250fa9..e9c7c33bc0e2 100644 --- a/InvenTree/importer/tasks.py +++ b/InvenTree/importer/tasks.py @@ -8,7 +8,7 @@ @transaction.atomic -def load_data(session_id: int): +def import_data(session_id: int): """Load data from the provided file. Attempt to load data from the provided file, and potentially handle any errors. @@ -20,39 +20,7 @@ def load_data(session_id: int): try: session = importer.models.DataImportSession.objects.get(pk=session_id) logger.info("Loading data from session ID '%s'", session_id) + session.import_data() except (ValueError, importer.models.DataImportSession.DoesNotExist): logger.error("Data import session with ID '%s' does not exist", session_id) return - - # Clear any existing data rows - session.rows.all().delete() - - df = importer.operations.load_data_file(session.data_file) - - if df is None: - # TODO: Log an error message against the import session - logger.error('Failed to load data file') - return - - headers = df.headers - - # TODO: Mark the session as "importing" - # session.status = importer.status_codes.DataImportStatusCode.IMPORTING.value - # session.save() - - row_objects = [] - - # Iterate through each "row" in the data file, and create a new DataImportRow object - for idx, row in enumerate(df): - row_data = dict(zip(headers, row)) - - row_objects.append( - importer.models.DataImportRow( - session=session, row_data=row_data, row_index=idx - ) - ) - - # Finally, create the DataImportRow objects - importer.models.DataImportRow.objects.bulk_create(row_objects) - - # TODO: Mark the import task as "completed" From 4863f206bf2e2d9f922a620a145535b7b2a64884 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 05:07:19 +0000 Subject: [PATCH 023/190] Update models - Add "columns" field to DataImportSession - Add "errors" field to DataImportRow --- InvenTree/importer/admin.py | 12 ++++ InvenTree/importer/migrations/0001_initial.py | 2 + InvenTree/importer/models.py | 70 ++++++++++++++++++- InvenTree/importer/serializers.py | 3 +- 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/InvenTree/importer/admin.py b/InvenTree/importer/admin.py index 2f1507a63e9c..6070ec1fd84d 100644 --- a/InvenTree/importer/admin.py +++ b/InvenTree/importer/admin.py @@ -13,6 +13,18 @@ class DataImportSessionAdmin(admin.ModelAdmin): list_filter = ['status'] + def get_readonly_fields(self, request, obj=None): + """Update the readonly fields for the admin interface.""" + fields = ['columns', 'status'] + + # Prevent data file from being edited after upload! + if obj: + fields += ['data_file'] + else: + fields += ['field_mapping'] + + return fields + @admin.register(importer.models.DataImportRow) class DataImportRowAdmin(admin.ModelAdmin): diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index 001f2fd8838e..81aa1100e2ee 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -27,6 +27,7 @@ class Migration(migrations.Migration): ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=GetExportFormats()), importer.validators.validate_data_file], verbose_name='Data File')), ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), + ('columns', models.JSONField(blank=True, null=True, verbose_name='Columns')), ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults')), ('field_mapping', models.JSONField(blank=True, null=True, verbose_name='Field Mapping')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), @@ -39,6 +40,7 @@ class Migration(migrations.Migration): ('row_index', models.PositiveIntegerField(default=0, verbose_name='Row Index')), ('row_data', models.JSONField(blank=True, null=True, verbose_name='Original row data')), ('data', models.JSONField(blank=True, null=True, verbose_name='Data')), + ('errors', models.JSONField(blank=True, null=True, verbose_name='Errors')), ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rows', to='importer.dataimportsession', verbose_name='Import Session')), ('complete', models.BooleanField(default=False, verbose_name='Complete')), ], diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 4de216bb6ecc..562117cbc70e 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -1,10 +1,13 @@ """Model definitions for the 'importer' app.""" from django.contrib.auth.models import User +from django.core.exceptions import ValidationError as DjangoValidationError from django.core.validators import FileExtensionValidator from django.db import models, transaction from django.utils.translation import gettext_lazy as _ +from rest_framework.exceptions import ValidationError as DRFValidationError + import importer.operations import importer.validators import InvenTree.helpers @@ -37,6 +40,7 @@ def save(self, *args, **kwargs): # New object - run initial setup self.status = DataImportStatusCode.INITIAL.value self.progress = 0 + self.extract_column_names() self.auto_assign_columns() self.save() @@ -56,6 +60,15 @@ def save(self, *args, **kwargs): ], ) + columns = models.JSONField(blank=True, null=True, verbose_name=_('Columns')) + + def extract_column_names(self): + """Extract column names from uploaded file. + + This method is called once, when the file is first uploaded. + """ + self.columns = importer.operations.extract_column_names(self.data_file) + model_type = models.CharField( blank=False, max_length=100, @@ -99,9 +112,7 @@ def serializer_fields(self, required=None, read_only=False): def auto_assign_columns(self): """Automatically assign columns based on the serializer fields.""" - from importer.operations import extract_column_names - - available_columns = extract_column_names(self.data_file) + available_columns = self.columns serializer_fields = self.serializer_fields() # Create a mapping of column names to serializer fields @@ -169,6 +180,11 @@ def import_data(self): self.status = DataImportStatusCode.PROCESSING.value self.save() + # Set initial data and errors for each row + for row in self.rows.all(): + row.extract_data(field_mapping=self.field_mapping) + row.validate() + @property def row_count(self): """Return the number of rows in the import session.""" @@ -209,4 +225,52 @@ class DataImportRow(models.Model): data = models.JSONField(blank=True, null=True, verbose_name=_('Data')) + errors = models.JSONField(blank=True, null=True, verbose_name=_('Errors')) + complete = models.BooleanField(default=False, verbose_name=_('Complete')) + + def extract_data(self, field_mapping: dict = None): + """Extract row data from the provided data dictionary.""" + if not field_mapping: + field_mapping = self.session.field_mapping + + data = {} + + # We have mapped column (file) to field (serializer) already + for field, col in field_mapping.items(): + data[field] = self.row_data.get(col, None) + + self.data = data + self.save() + + def set_errors(self, errors: dict) -> None: + """Set the error messages for this row.""" + self.errors = errors + self.save() + + def validate(self, raise_exception=True) -> bool: + """Validate the data in this row against the linked serializer. + + Returns: + True if the data is valid, False otherwise + + Raises: + ValidationError: If the linked serializer is not valid + """ + serializer_class = self.session.serializer if self.session else None + + if not serializer_class: + self.set_errors({ + 'non_field_errors': 'No serializer class linked to this import session' + }) + return + + result = False + + try: + serializer = serializer_class(data=self.data) + result = serializer.is_valid(raise_exception=raise_exception) + except (DjangoValidationError, DRFValidationError) as e: + self.set_errors(e.detail) + + return result diff --git a/InvenTree/importer/serializers.py b/InvenTree/importer/serializers.py index 40d7c794e7df..920e7ececeec 100644 --- a/InvenTree/importer/serializers.py +++ b/InvenTree/importer/serializers.py @@ -22,12 +22,13 @@ class Meta: 'model_type', 'status', 'user', + 'columns', 'field_defaults', 'field_mapping', 'row_count', 'completed_row_count', ] - read_only_fields = ['pk', 'user', 'status'] + read_only_fields = ['pk', 'user', 'status', 'columns'] row_count = serializers.IntegerField(read_only=True) completed_row_count = serializers.IntegerField(read_only=True) From 657500bcfd7cc6bf7055d7e24715d45543676efc Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 06:01:51 +0000 Subject: [PATCH 024/190] Move field mapping to a new model type - Simpler validation --- InvenTree/build/models.py | 2 +- InvenTree/importer/admin.py | 8 ++ InvenTree/importer/migrations/0001_initial.py | 13 ++- InvenTree/importer/models.py | 81 +++++++++++++++++-- 4 files changed, 95 insertions(+), 9 deletions(-) diff --git a/InvenTree/build/models.py b/InvenTree/build/models.py index 43308ad9bac4..eaa59cf79c63 100644 --- a/InvenTree/build/models.py +++ b/InvenTree/build/models.py @@ -1287,7 +1287,7 @@ class BuildLine(InvenTree.models.InvenTreeModel): """ class Meta: - """Model meta options""" + """Model meta options.""" unique_together = [ ('build', 'bom_item'), ] diff --git a/InvenTree/importer/admin.py b/InvenTree/importer/admin.py index 6070ec1fd84d..9f19ffcdd294 100644 --- a/InvenTree/importer/admin.py +++ b/InvenTree/importer/admin.py @@ -5,6 +5,12 @@ import importer.models +class DataImportColumnMapAdmin(admin.TabularInline): + """Inline admin for DataImportColumnMap model.""" + + model = importer.models.DataImportColumnMap + + @admin.register(importer.models.DataImportSession) class DataImportSessionAdmin(admin.ModelAdmin): """Admin interface for the DataImportSession model.""" @@ -13,6 +19,8 @@ class DataImportSessionAdmin(admin.ModelAdmin): list_filter = ['status'] + inlines = [DataImportColumnMapAdmin] + def get_readonly_fields(self, request, obj=None): """Update the readonly fields for the admin interface.""" fields = ['columns', 'status'] diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index 81aa1100e2ee..c9b6b0880374 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -29,10 +29,21 @@ class Migration(migrations.Migration): ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), ('columns', models.JSONField(blank=True, null=True, verbose_name='Columns')), ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults')), - ('field_mapping', models.JSONField(blank=True, null=True, verbose_name='Field Mapping')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], ), + migrations.CreateModel( + name='DataImportColumnMap', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('column', models.CharField(max_length=100, verbose_name='Column')), + ('field', models.CharField(max_length=100, verbose_name='Field')), + ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='column_mappings', to='importer.dataimportsession', verbose_name='Import Session')), + ], + options={ + 'unique_together': {('session', 'column'), ('session', 'field')}, + }, + ), migrations.CreateModel( name='DataImportRow', fields=[ diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 562117cbc70e..ab9239e905b4 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -89,9 +89,19 @@ def extract_column_names(self): blank=True, null=True, verbose_name=_('Field Defaults') ) - field_mapping = models.JSONField( - blank=True, null=True, verbose_name=_('Field Mapping') - ) + @property + def field_mapping(self): + """Construct a dict of field mappings for this import session. + + Returns: A dict of field: column mappings + """ + mapping = {} + + if self.column_mappings.exists(): + for map in self.column_mappings.all(): + mapping[map.field] = map.column + + return mapping @property def serializer(self): @@ -115,7 +125,9 @@ def auto_assign_columns(self): available_columns = self.columns serializer_fields = self.serializer_fields() - # Create a mapping of column names to serializer fields + # Remove any existing mappings + + # Create an initial mapping of column names to serializer fields column_map = {} for name, field in serializer_fields.items(): @@ -127,9 +139,13 @@ def auto_assign_columns(self): elif label in available_columns: column = label - column_map[name] = column - - self.field_mapping = column_map + if column is not None: + try: + DataImportColumnMap.objects.create( + session=self, field=name, column=column + ) + except Exception: + pass def trigger_data_import(self): """Trigger the data import process for this session. @@ -199,6 +215,57 @@ def completed_row_count(self): return self.rows.filter(complete=True).count() / self.row_count * 100 +class DataImportColumnMap(models.Model): + """Database model representing a mapping between a file column and serializer field. + + - Each row maps a "column" (in the import file) to a "field" (in the serializer) + - Column must exist in the file + - Field must exist in the serializer (and not be read-only) + """ + + class Meta: + """Model meta options.""" + + unique_together = [('session', 'column'), ('session', 'field')] + + def clean(self): + """Validate the column mapping.""" + super().clean() + + if not self.session: + raise DjangoValidationError({ + 'session': _('Column mapping must be linked to a valid import session') + }) + + if self.column not in self.session.columns: + raise DjangoValidationError({ + 'column': _('Column does not exist in the data file') + }) + + fields = self.session.serializer_fields(read_only=None) + + if self.field not in fields: + raise DjangoValidationError({ + 'field': _('Field does not exist in the target model') + }) + + field_def = fields[self.field] + + if field_def.read_only: + raise DjangoValidationError({'field': _('Selected field is read-only')}) + + session = models.ForeignKey( + DataImportSession, + on_delete=models.CASCADE, + verbose_name=_('Import Session'), + related_name='column_mappings', + ) + + column = models.CharField(max_length=100, verbose_name=_('Column')) + + field = models.CharField(max_length=100, verbose_name=_('Field')) + + class DataImportRow(models.Model): """Database model representing a single row in a data import session. From 122e2d348734ab1b29513d23e642619fe98d9ffa Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 06:21:58 +0000 Subject: [PATCH 025/190] Save "valid" status for each data row --- InvenTree/importer/admin.py | 4 ++++ InvenTree/importer/migrations/0001_initial.py | 1 + InvenTree/importer/models.py | 21 ++++++++++++------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/InvenTree/importer/admin.py b/InvenTree/importer/admin.py index 9f19ffcdd294..c18aa2037347 100644 --- a/InvenTree/importer/admin.py +++ b/InvenTree/importer/admin.py @@ -39,3 +39,7 @@ class DataImportRowAdmin(admin.ModelAdmin): """Admin interface for the DataImportRow model.""" list_display = ['id', 'session', 'row_index'] + + def get_readonly_fields(self, request, obj=None): + """Return the readonly fields for the admin interface.""" + return ['session', 'row_index', 'row_data', 'errors', 'valid'] diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index c9b6b0880374..67f070534a24 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -53,6 +53,7 @@ class Migration(migrations.Migration): ('data', models.JSONField(blank=True, null=True, verbose_name='Data')), ('errors', models.JSONField(blank=True, null=True, verbose_name='Errors')), ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rows', to='importer.dataimportsession', verbose_name='Import Session')), + ('valid', models.BooleanField(default=False, verbose_name='Valid')), ('complete', models.BooleanField(default=False, verbose_name='Complete')), ], ), diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index ab9239e905b4..8c7be9caf5c6 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -277,6 +277,11 @@ class DataImportRow(models.Model): status: IntegerField for the status of the row import """ + def save(self, *args, **kwargs): + """Save the DataImportRow object.""" + self.valid = self.validate() + super().save(*args, **kwargs) + session = models.ForeignKey( DataImportSession, on_delete=models.CASCADE, @@ -294,6 +299,8 @@ class DataImportRow(models.Model): errors = models.JSONField(blank=True, null=True, verbose_name=_('Errors')) + valid = models.BooleanField(default=False, verbose_name=_('Valid')) + complete = models.BooleanField(default=False, verbose_name=_('Complete')) def extract_data(self, field_mapping: dict = None): @@ -310,12 +317,7 @@ def extract_data(self, field_mapping: dict = None): self.data = data self.save() - def set_errors(self, errors: dict) -> None: - """Set the error messages for this row.""" - self.errors = errors - self.save() - - def validate(self, raise_exception=True) -> bool: + def validate(self) -> bool: """Validate the data in this row against the linked serializer. Returns: @@ -336,8 +338,11 @@ def validate(self, raise_exception=True) -> bool: try: serializer = serializer_class(data=self.data) - result = serializer.is_valid(raise_exception=raise_exception) + result = serializer.is_valid(raise_exception=True) except (DjangoValidationError, DRFValidationError) as e: - self.set_errors(e.detail) + self.errors = e.detail + + if result: + self.errors = None return result From 6d43109875a7d063e02dc5134384e5eceb0b3623 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 06:30:52 +0000 Subject: [PATCH 026/190] Include session defaults when validating row data --- InvenTree/importer/models.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 8c7be9caf5c6..749b93261c88 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -317,6 +317,24 @@ def extract_data(self, field_mapping: dict = None): self.data = data self.save() + def serializer_data(self): + """Construct data object to be sent to the serializer. + + Note that we also use the "default" values provided by the import session + """ + session_defaults = self.session.field_defaults or {} + + return {**session_defaults, **self.data} + + def construct_serializer(self): + """Construct a serializer object for this row.""" + serializer_class = self.session.serializer + + if not serializer_class: + return None + + return serializer_class(data=self.serializer_data) + def validate(self) -> bool: """Validate the data in this row against the linked serializer. @@ -326,18 +344,17 @@ def validate(self) -> bool: Raises: ValidationError: If the linked serializer is not valid """ - serializer_class = self.session.serializer if self.session else None + serializer = self.construct_serializer() - if not serializer_class: - self.set_errors({ + if not serializer: + self.errors = { 'non_field_errors': 'No serializer class linked to this import session' - }) + } return result = False try: - serializer = serializer_class(data=self.data) result = serializer.is_valid(raise_exception=True) except (DjangoValidationError, DRFValidationError) as e: self.errors = e.detail From c6e5e74fb673356921e2fe092ab577fafa023ee2 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 10:03:00 +0000 Subject: [PATCH 027/190] Update content_excludes - Ignore importer models in import/export --- tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tasks.py b/tasks.py index 0292a14f75e7..23d8ba557843 100644 --- a/tasks.py +++ b/tasks.py @@ -86,6 +86,9 @@ def content_excludes( 'common.notificationentry', 'common.notificationmessage', 'user_sessions.session', + 'importer.dataimportsession', + 'importer.dataimportcolumnmap', + 'importer.dataimportrow', ] # Optionally exclude user auth data From e688eb21fd3b71168b7c11731607911a65a53ce4 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 10:08:34 +0000 Subject: [PATCH 028/190] Remove port from ALLOWED_HOST entries --- InvenTree/InvenTree/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/InvenTree/InvenTree/settings.py b/InvenTree/InvenTree/settings.py index e12d80c0548f..90bd8c415218 100644 --- a/InvenTree/InvenTree/settings.py +++ b/InvenTree/InvenTree/settings.py @@ -1016,7 +1016,10 @@ # Ensure that the ALLOWED_HOSTS do not contain any scheme info for i, host in enumerate(ALLOWED_HOSTS): if '://' in host: - ALLOWED_HOSTS[i] = host.split('://')[1] + ALLOWED_HOSTS[i] = host = host.split('://')[1] + + if ':' in host: + ALLOWED_HOSTS[i] = host = host.split(':')[0] # List of trusted origins for unsafe requests # Ref: https://docs.djangoproject.com/en/4.2/ref/settings/#csrf-trusted-origins From fc4832d97ef6fab30cc0065f89c605d734a56ee2 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 10:11:08 +0000 Subject: [PATCH 029/190] Skip table events for importer models --- InvenTree/plugin/base/event/events.py | 1 + 1 file changed, 1 insertion(+) diff --git a/InvenTree/plugin/base/event/events.py b/InvenTree/plugin/base/event/events.py index 752a7a8d8566..feafc86af2fb 100644 --- a/InvenTree/plugin/base/event/events.py +++ b/InvenTree/plugin/base/event/events.py @@ -135,6 +135,7 @@ def allow_table_event(table_name): 'socialaccount_', 'user_', 'users_', + 'importer_', ] if any(table_name.startswith(prefix) for prefix in ignore_prefixes): From 0f30ae87af271e17fcc57800f32198bcab0b2d98 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 10:17:31 +0000 Subject: [PATCH 030/190] Bug fixes --- InvenTree/InvenTree/tasks.py | 4 ++-- InvenTree/importer/models.py | 12 ++++++++---- InvenTree/importer/tasks.py | 3 --- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/InvenTree/InvenTree/tasks.py b/InvenTree/InvenTree/tasks.py index 4356509e12e3..a9c1569b5217 100644 --- a/InvenTree/InvenTree/tasks.py +++ b/InvenTree/InvenTree/tasks.py @@ -264,8 +264,8 @@ def offload_task( _func(*args, **kwargs) except Exception as exc: log_error('InvenTree.offload_task') - raise_warning(f"WARNING: '{taskname}' not started due to {str(exc)}") - return False + raise_warning(f"WARNING: '{taskname}' failed due to {str(exc)}") + raise exc # Finally, task either completed successfully or was offloaded return True diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 749b93261c88..ea4a15764d93 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -1,18 +1,23 @@ """Model definitions for the 'importer' app.""" +import logging + from django.contrib.auth.models import User from django.core.exceptions import ValidationError as DjangoValidationError from django.core.validators import FileExtensionValidator -from django.db import models, transaction +from django.db import models from django.utils.translation import gettext_lazy as _ from rest_framework.exceptions import ValidationError as DRFValidationError import importer.operations +import importer.tasks import importer.validators import InvenTree.helpers from importer.status_codes import DataImportStatusCode +logger = logging.getLogger('inventree') + class DataImportSession(models.Model): """Database model representing a data import session. @@ -152,10 +157,9 @@ def trigger_data_import(self): Offloads the task to the background worker process. """ - from importer.tasks import import_data from InvenTree.tasks import offload_task - offload_task(import_data, self.pk) + offload_task(importer.tasks.import_data, self.pk) def import_data(self): """Perform the data import process for this session.""" @@ -333,7 +337,7 @@ def construct_serializer(self): if not serializer_class: return None - return serializer_class(data=self.serializer_data) + return serializer_class(data=self.serializer_data()) def validate(self) -> bool: """Validate the data in this row against the linked serializer. diff --git a/InvenTree/importer/tasks.py b/InvenTree/importer/tasks.py index e9c7c33bc0e2..224610094a3c 100644 --- a/InvenTree/importer/tasks.py +++ b/InvenTree/importer/tasks.py @@ -2,12 +2,9 @@ import logging -from django.db import transaction - logger = logging.getLogger('inventree') -@transaction.atomic def import_data(session_id: int): """Load data from the provided file. From e1587a9db9227396f3f8bdb8555513fe4bbde0cc Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 10:35:41 +0000 Subject: [PATCH 031/190] Serializer updates --- InvenTree/importer/models.py | 23 +++++++++++++++++++---- InvenTree/importer/serializers.py | 23 +++++++++++++++++++++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index ea4a15764d93..679751cc6038 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -246,18 +246,22 @@ def clean(self): 'column': _('Column does not exist in the data file') }) - fields = self.session.serializer_fields(read_only=None) + field_def = self.field_definition - if self.field not in fields: + if not field_def: raise DjangoValidationError({ 'field': _('Field does not exist in the target model') }) - field_def = fields[self.field] - if field_def.read_only: raise DjangoValidationError({'field': _('Selected field is read-only')}) + @property + def field_definition(self): + """Return the field definition associated with this column mapping.""" + fields = self.session.serializer_fields(read_only=None) + return fields.get(self.field, None) + session = models.ForeignKey( DataImportSession, on_delete=models.CASCADE, @@ -269,6 +273,17 @@ def clean(self): field = models.CharField(max_length=100, verbose_name=_('Field')) + @property + def label(self): + """Extract the 'label' associated with the mapped field.""" + field_def = self.field_definition + + if field_def: + return field_def.label + + # Default to the field name + return self.field + class DataImportRow(models.Model): """Database model representing a single row in a data import session. diff --git a/InvenTree/importer/serializers.py b/InvenTree/importer/serializers.py index 920e7ececeec..f767fcffb2b5 100644 --- a/InvenTree/importer/serializers.py +++ b/InvenTree/importer/serializers.py @@ -5,7 +5,22 @@ from rest_framework import serializers import importer.models -from InvenTree.serializers import InvenTreeModelSerializer +from InvenTree.serializers import ( + InvenTreeAttachmentSerializerField, + InvenTreeModelSerializer, +) + + +class DataImportColumnMapSerializer(InvenTreeModelSerializer): + """Serializer for the DataImportColumnMap model.""" + + class Meta: + """Meta class options for the serializer.""" + + model = importer.models.DataImportColumnMap + fields = ['pk', 'session', 'column', 'field', 'label'] + + label = serializers.CharField(read_only=True) class DataImportSessionSerializer(InvenTreeModelSerializer): @@ -23,16 +38,20 @@ class Meta: 'status', 'user', 'columns', + 'column_mappings', 'field_defaults', - 'field_mapping', 'row_count', 'completed_row_count', ] read_only_fields = ['pk', 'user', 'status', 'columns'] + data_file = InvenTreeAttachmentSerializerField(read_only=True) + row_count = serializers.IntegerField(read_only=True) completed_row_count = serializers.IntegerField(read_only=True) + column_mappings = DataImportColumnMapSerializer(many=True, read_only=True) + def create(self, validated_data): """Override create method for this serializer. From 31174174eeb17cae35fd70a71707d14c111721d9 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 10:54:16 +0000 Subject: [PATCH 032/190] Add more endpoints - DataImportColumnMappingList - DataImportRowList --- InvenTree/importer/api.py | 43 +++++++++++++++++++++++++++++-- InvenTree/importer/serializers.py | 29 +++++++++++++++++++++ 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/InvenTree/importer/api.py b/InvenTree/importer/api.py index f238503680d0..68a526e6b66b 100644 --- a/InvenTree/importer/api.py +++ b/InvenTree/importer/api.py @@ -4,7 +4,8 @@ import importer.models import importer.serializers -from InvenTree.mixins import ListCreateAPI +from InvenTree.filters import SEARCH_ORDER_FILTER +from InvenTree.mixins import ListAPI, ListCreateAPI class DataImportSessionList(ListCreateAPI): @@ -14,11 +15,49 @@ class DataImportSessionList(ListCreateAPI): serializer_class = importer.serializers.DataImportSessionSerializer +class DataImportColumnMappingList(ListCreateAPI): + """API endpoint for accessing a list of DataImportColumnMap objects.""" + + queryset = importer.models.DataImportColumnMap.objects.all() + serializer_class = importer.serializers.DataImportColumnMapSerializer + + filter_backends = SEARCH_ORDER_FILTER + + filterset_fields = ['session'] + + +class DataImportRowList(ListAPI): + """API endpoint for accessing a list of DataImportRow objects.""" + + queryset = importer.models.DataImportRow.objects.all() + serializer_class = importer.serializers.DataImportRowSerializer + + filter_backends = SEARCH_ORDER_FILTER + + filterset_fields = ['session'] + + ordering_fields = ['pk', 'row_index'] + + ordering = 'row_index' + + importer_api_urls = [ path( 'session/', include([ path('', DataImportSessionList.as_view(), name='importer-session-list') ]), - ) + ), + path( + 'column-mapping/', + include([ + path( + '', DataImportColumnMappingList.as_view(), name='importer-mapping-list' + ) + ]), + ), + path( + 'row/', + include([path('', DataImportRowList.as_view(), name='importer-row-list')]), + ), ] diff --git a/InvenTree/importer/serializers.py b/InvenTree/importer/serializers.py index f767fcffb2b5..599f57dae7b7 100644 --- a/InvenTree/importer/serializers.py +++ b/InvenTree/importer/serializers.py @@ -66,3 +66,32 @@ def create(self, validated_data): session.save() return session + + +class DataImportRowSerializer(InvenTreeModelSerializer): + """Serializer for the DataImportRow model.""" + + class Meta: + """Meta class options for the serializer.""" + + model = importer.models.DataImportRow + fields = [ + 'pk', + 'session', + 'row_index', + 'row_data', + 'data', + 'errors', + 'valid', + 'complete', + ] + + read_only_fields = [ + 'pk', + 'session', + 'row_index', + 'data', + 'errors', + 'valid', + 'complete', + ] From ddb23a125a150f0fc3dd28c8f1dd981b91e82cd0 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 11:00:05 +0000 Subject: [PATCH 033/190] further updates: - Add 'get_api_url' method - Handle case where --- InvenTree/InvenTree/metadata.py | 5 ++++- InvenTree/importer/api.py | 8 +++++--- InvenTree/importer/models.py | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/InvenTree/InvenTree/metadata.py b/InvenTree/InvenTree/metadata.py index 65097ed75814..af7debe6612d 100644 --- a/InvenTree/InvenTree/metadata.py +++ b/InvenTree/InvenTree/metadata.py @@ -280,8 +280,11 @@ def get_field_info(self, field): # Special case for 'user' model if field_info['model'] == 'user': field_info['api_url'] = '/api/user/' - else: + elif hasattr(model, 'get_api_url'): field_info['api_url'] = model.get_api_url() + else: + logger.warning("'get_api_url' method not defined for %s", model) + field_info['api_url'] = getattr(model, 'api_url', None) # Add more metadata about dependent fields if field_info['type'] == 'dependent field': diff --git a/InvenTree/importer/api.py b/InvenTree/importer/api.py index 68a526e6b66b..efc1fe925ddc 100644 --- a/InvenTree/importer/api.py +++ b/InvenTree/importer/api.py @@ -45,19 +45,21 @@ class DataImportRowList(ListAPI): path( 'session/', include([ - path('', DataImportSessionList.as_view(), name='importer-session-list') + path('', DataImportSessionList.as_view(), name='api-importer-session-list') ]), ), path( 'column-mapping/', include([ path( - '', DataImportColumnMappingList.as_view(), name='importer-mapping-list' + '', + DataImportColumnMappingList.as_view(), + name='api-importer-mapping-list', ) ]), ), path( 'row/', - include([path('', DataImportRowList.as_view(), name='importer-row-list')]), + include([path('', DataImportRowList.as_view(), name='api-importer-row-list')]), ), ] diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 679751cc6038..2a40e4e518fb 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -6,6 +6,7 @@ from django.core.exceptions import ValidationError as DjangoValidationError from django.core.validators import FileExtensionValidator from django.db import models +from django.urls import reverse from django.utils.translation import gettext_lazy as _ from rest_framework.exceptions import ValidationError as DRFValidationError @@ -33,6 +34,11 @@ class DataImportSession(models.Model): field_overrides: JSONField for field overrides (e.g. custom field values) """ + @staticmethod + def get_api_url(): + """Return the API URL associated with the DataImportSession model.""" + return reverse('api-importer-session-list') + def save(self, *args, **kwargs): """Save the DataImportSession object.""" initial = self.pk is None @@ -227,6 +233,11 @@ class DataImportColumnMap(models.Model): - Field must exist in the serializer (and not be read-only) """ + @staticmethod + def get_api_url(): + """Return the API URL associated with the DataImportColumnMap model.""" + return reverse('api-importer-mapping-list') + class Meta: """Model meta options.""" @@ -296,6 +307,11 @@ class DataImportRow(models.Model): status: IntegerField for the status of the row import """ + @staticmethod + def get_api_url(): + """Return the API URL associated with the DataImportRow model.""" + return reverse('api-importer-row-list') + def save(self, *args, **kwargs): """Save the DataImportRow object.""" self.valid = self.validate() From a2ece1512a6dba0c4c79f0e3f0b8aae14848fd6f Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 11:11:58 +0000 Subject: [PATCH 034/190] Expose "available fields" to the DataImportSession serializer Uses the (already available) inventree metadata middleware --- InvenTree/InvenTree/metadata.py | 3 ++- InvenTree/importer/models.py | 14 ++++++++++++++ InvenTree/importer/serializers.py | 3 +++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/InvenTree/InvenTree/metadata.py b/InvenTree/InvenTree/metadata.py index af7debe6612d..47f7feb59313 100644 --- a/InvenTree/InvenTree/metadata.py +++ b/InvenTree/InvenTree/metadata.py @@ -199,7 +199,8 @@ def get_serializer_info(self, serializer): if instance is None and model_class is not None: # Attempt to find the instance based on kwargs lookup - kwargs = getattr(self.view, 'kwargs', None) + view = getattr(self, 'view', None) + kwargs = getattr(view, 'kwargs', None) if view else None if kwargs: pk = None diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 2a40e4e518fb..e50d7c3b66fe 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -224,6 +224,20 @@ def completed_row_count(self): return self.rows.filter(complete=True).count() / self.row_count * 100 + def available_fields(self): + """Returns information on the available fields. + + - This method is designed to be introspected by the frontend, for rendering the various fields. + - We make use of the InvenTree.metadata module to provide extra information about the fields. + """ + from InvenTree.metadata import InvenTreeMetadata + + metadata = InvenTreeMetadata() + if serializer := self.serializer: + return metadata.get_serializer_info(serializer(data={})) + else: + return {} + class DataImportColumnMap(models.Model): """Database model representing a mapping between a file column and serializer field. diff --git a/InvenTree/importer/serializers.py b/InvenTree/importer/serializers.py index 599f57dae7b7..8f4f4fb5a1e6 100644 --- a/InvenTree/importer/serializers.py +++ b/InvenTree/importer/serializers.py @@ -35,6 +35,7 @@ class Meta: 'timestamp', 'data_file', 'model_type', + 'available_fields', 'status', 'user', 'columns', @@ -47,6 +48,8 @@ class Meta: data_file = InvenTreeAttachmentSerializerField(read_only=True) + available_fields = serializers.JSONField(read_only=True) + row_count = serializers.IntegerField(read_only=True) completed_row_count = serializers.IntegerField(read_only=True) From 8baf848c3b98c84f3766939399d2863c6ba6ec2e Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 11:26:22 +0000 Subject: [PATCH 035/190] Add detail endpoints --- InvenTree/importer/api.py | 46 +++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/InvenTree/importer/api.py b/InvenTree/importer/api.py index efc1fe925ddc..5022923f7ab3 100644 --- a/InvenTree/importer/api.py +++ b/InvenTree/importer/api.py @@ -5,7 +5,7 @@ import importer.models import importer.serializers from InvenTree.filters import SEARCH_ORDER_FILTER -from InvenTree.mixins import ListAPI, ListCreateAPI +from InvenTree.mixins import ListAPI, ListCreateAPI, RetrieveUpdateDestroyAPI class DataImportSessionList(ListCreateAPI): @@ -15,6 +15,13 @@ class DataImportSessionList(ListCreateAPI): serializer_class = importer.serializers.DataImportSessionSerializer +class DataImportSessionDetail(RetrieveUpdateDestroyAPI): + """Detail endpoint for a single DataImportSession object.""" + + queryset = importer.models.DataImportSession.objects.all() + serializer_class = importer.serializers.DataImportSessionSerializer + + class DataImportColumnMappingList(ListCreateAPI): """API endpoint for accessing a list of DataImportColumnMap objects.""" @@ -26,6 +33,13 @@ class DataImportColumnMappingList(ListCreateAPI): filterset_fields = ['session'] +class DataImportColumnMappingDetail(RetrieveUpdateDestroyAPI): + """Detail endpoint for a single DataImportColumnMap object.""" + + queryset = importer.models.DataImportColumnMap.objects.all() + serializer_class = importer.serializers.DataImportColumnMapSerializer + + class DataImportRowList(ListAPI): """API endpoint for accessing a list of DataImportRow objects.""" @@ -41,25 +55,49 @@ class DataImportRowList(ListAPI): ordering = 'row_index' +class DataImportRowDetail(RetrieveUpdateDestroyAPI): + """Detail endpoint for a single DataImportRow object.""" + + queryset = importer.models.DataImportRow.objects.all() + serializer_class = importer.serializers.DataImportRowSerializer + + importer_api_urls = [ path( 'session/', include([ - path('', DataImportSessionList.as_view(), name='api-importer-session-list') + path( + '/', + DataImportSessionDetail.as_view(), + name='api-import-session-detail', + ), + path('', DataImportSessionList.as_view(), name='api-importer-session-list'), ]), ), path( 'column-mapping/', include([ + path( + '/', + DataImportColumnMappingDetail.as_view(), + name='api-importer-mapping-detail', + ), path( '', DataImportColumnMappingList.as_view(), name='api-importer-mapping-list', - ) + ), ]), ), path( 'row/', - include([path('', DataImportRowList.as_view(), name='api-importer-row-list')]), + include([ + path( + '/', + DataImportRowDetail.as_view(), + name='api-importer-row-detail', + ), + path('', DataImportRowList.as_view(), name='api-importer-row-list'), + ]), ), ] From 2a797a0673a8fa2463222fc3fd31af9449183888 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 11:26:53 +0000 Subject: [PATCH 036/190] Clear existing column mappings --- InvenTree/importer/models.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index e50d7c3b66fe..4c17380e9972 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -137,9 +137,7 @@ def auto_assign_columns(self): serializer_fields = self.serializer_fields() # Remove any existing mappings - - # Create an initial mapping of column names to serializer fields - column_map = {} + self.column_mappings.all().delete() for name, field in serializer_fields.items(): column = None From fa0b1db733d26bbdcc3ff403ff5f506ccbf44a33 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 11:54:45 +0000 Subject: [PATCH 037/190] Add endpoint for accepting column mappings --- InvenTree/importer/api.py | 38 ++++++++++++- InvenTree/importer/migrations/0001_initial.py | 2 +- InvenTree/importer/models.py | 53 ++++++++++++++++--- InvenTree/importer/validators.py | 9 ++++ 4 files changed, 93 insertions(+), 9 deletions(-) diff --git a/InvenTree/importer/api.py b/InvenTree/importer/api.py index 5022923f7ab3..bfeab9e3f34a 100644 --- a/InvenTree/importer/api.py +++ b/InvenTree/importer/api.py @@ -1,7 +1,13 @@ """API endpoints for the importer app.""" +from django.shortcuts import get_object_or_404 from django.urls import include, path +from drf_spectacular.utils import extend_schema +from rest_framework import permissions +from rest_framework.response import Response +from rest_framework.views import APIView + import importer.models import importer.serializers from InvenTree.filters import SEARCH_ORDER_FILTER @@ -22,6 +28,24 @@ class DataImportSessionDetail(RetrieveUpdateDestroyAPI): serializer_class = importer.serializers.DataImportSessionSerializer +class DataImportSessionAcceptFields(APIView): + """API endpoint to accept the field mapping for a DataImportSession.""" + + permission_classes = [permissions.IsAuthenticated] + + @extend_schema( + responses={200: importer.serializers.DataImportSessionSerializer(many=False)} + ) + def post(self, request, pk): + """Accept the field mapping for a DataImportSession.""" + session = get_object_or_404(importer.models.DataImportSession, pk=pk) + + # Attempt to accept the mapping (may raise an exception if the mapping is invalid) + session.accept_mapping() + + return Response(importer.serializers.DataImportSessionSerializer(session).data) + + class DataImportColumnMappingList(ListCreateAPI): """API endpoint for accessing a list of DataImportColumnMap objects.""" @@ -68,8 +92,18 @@ class DataImportRowDetail(RetrieveUpdateDestroyAPI): include([ path( '/', - DataImportSessionDetail.as_view(), - name='api-import-session-detail', + include([ + path( + 'accept_fields/', + DataImportSessionAcceptFields.as_view(), + name='api-import-session-accept-fields', + ), + path( + '', + DataImportSessionDetail.as_view(), + name='api-import-session-detail', + ), + ]), ), path('', DataImportSessionList.as_view(), name='api-importer-session-list'), ]), diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index 67f070534a24..b42d6d0e68cd 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -28,7 +28,7 @@ class Migration(migrations.Migration): ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), ('columns', models.JSONField(blank=True, null=True, verbose_name='Columns')), - ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults')), + ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults', validators=[importer.validators.validate_field_defaults])), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], ), diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 4c17380e9972..4328a8864243 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -31,7 +31,7 @@ class DataImportSession(models.Model): status: IntegerField for the status of the import session user: ForeignKey to the User who initiated the import data_columns: JSONField for the data columns in the import file (mapped to database columns) - field_overrides: JSONField for field overrides (e.g. custom field values) + field_defaults: JSONField for field overrides (e.g. custom field values) """ @staticmethod @@ -97,7 +97,10 @@ def extract_column_names(self): ) field_defaults = models.JSONField( - blank=True, null=True, verbose_name=_('Field Defaults') + blank=True, + null=True, + verbose_name=_('Field Defaults'), + validators=[importer.validators.validate_field_defaults], ) @property @@ -156,6 +159,40 @@ def auto_assign_columns(self): except Exception: pass + self.status = DataImportStatusCode.MAPPING.value + + def accept_mapping(self): + """Accept current mapping configuration. + + - Validate that the current column mapping is correct + - Trigger the data import process + """ + # First, we need to ensure that all the *required* columns have been mapped + required_fields = self.serializer_fields(required=True).keys() + field_defaults = self.field_defaults or {} + + missing_fields = [] + + for field in required_fields: + # An explicit mapping exists + if self.column_mappings.filter(field=field).exists(): + continue + + # A default value exists + if field in field_defaults: + continue + + missing_fields.append(field) + + if len(missing_fields) > 0: + raise DjangoValidationError({ + 'error': _('Some required fields have not been mapped'), + 'fields': missing_fields, + }) + + # No errors, so trigger the data import process + self.trigger_data_import() + def trigger_data_import(self): """Trigger the data import process for this session. @@ -163,6 +200,10 @@ def trigger_data_import(self): """ from InvenTree.tasks import offload_task + # Mark the import task status as "IMPORTING" + self.status = DataImportStatusCode.IMPORTING.value + self.save() + offload_task(importer.tasks.import_data, self.pk) def import_data(self): @@ -183,10 +224,6 @@ def import_data(self): row_objects = [] - # Mark the import task status as "IMPORTING" - self.status = DataImportStatusCode.IMPORTING.value - self.save() - # Iterate through each "row" in the data file, and create a new DataImportRow object for idx, row in enumerate(df): row_data = dict(zip(headers, row)) @@ -371,6 +408,10 @@ def serializer_data(self): """ session_defaults = self.session.field_defaults or {} + print('serializer_data:') + print('- session_defaults:', session_defaults, ':', type(session_defaults)) + print('- self.data:', self.data, ':', type(self.data)) + return {**session_defaults, **self.data} def construct_serializer(self): diff --git a/InvenTree/importer/validators.py b/InvenTree/importer/validators.py index 4f6ab8cb5814..ee50f4017387 100644 --- a/InvenTree/importer/validators.py +++ b/InvenTree/importer/validators.py @@ -38,3 +38,12 @@ def validate_importer_model_type(value): if value not in supported_models().keys(): raise ValidationError(f"Unsupported model type '{value}'") + + +def validate_field_defaults(value): + """Validate that the provided value is a valid dict.""" + if value is None: + return + + if type(value) is not dict: + raise ValidationError(_('Field defaults must be a dictionary')) From 4bc70bc87577b4668bcb8094126154ea1b81d3ec Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 12:05:14 +0000 Subject: [PATCH 038/190] Add API endpoint exposing available importer serializers --- InvenTree/importer/api.py | 24 ++++++++++++++++++++++++ InvenTree/part/models.py | 5 +++++ 2 files changed, 29 insertions(+) diff --git a/InvenTree/importer/api.py b/InvenTree/importer/api.py index bfeab9e3f34a..263e9625d40e 100644 --- a/InvenTree/importer/api.py +++ b/InvenTree/importer/api.py @@ -9,11 +9,34 @@ from rest_framework.views import APIView import importer.models +import importer.registry import importer.serializers from InvenTree.filters import SEARCH_ORDER_FILTER from InvenTree.mixins import ListAPI, ListCreateAPI, RetrieveUpdateDestroyAPI +class DataImporterModelList(APIView): + """API endpoint for displaying a list of models available for import.""" + + permission_classes = [permissions.IsAuthenticated] + + def get(self, request): + """Return a list of models available for import.""" + models = [] + + for serializer in importer.registry.get_supported_serializers(): + model = serializer.Meta.model + url = model.get_api_url() if hasattr(model, 'get_api_url') else None + + models.append({ + 'serializer': str(serializer.__name__), + 'model_type': model.__name__.lower(), + 'api_url': url, + }) + + return Response(models) + + class DataImportSessionList(ListCreateAPI): """API endpoint for accessing a list of DataImportSession objects.""" @@ -87,6 +110,7 @@ class DataImportRowDetail(RetrieveUpdateDestroyAPI): importer_api_urls = [ + path('models/', DataImporterModelList.as_view(), name='api-importer-model-list'), path( 'session/', include([ diff --git a/InvenTree/part/models.py b/InvenTree/part/models.py index 2f2b34a409a4..3d4bea1d9ee0 100644 --- a/InvenTree/part/models.py +++ b/InvenTree/part/models.py @@ -3817,6 +3817,11 @@ class PartCategoryParameterTemplate(InvenTree.models.InvenTreeMetadataModel): category """ + @staticmethod + def get_api_url(): + """Return the API endpoint URL associated with the PartCategoryParameterTemplate model.""" + return reverse('api-part-category-parameter-list') + class Meta: """Metaclass providing extra model definition.""" From 37297a3022254a5c7fa3e137ce16708a04507e4f Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 12:12:02 +0000 Subject: [PATCH 039/190] Add simple playground area for testing data importer --- .../components/importer/ImporterDrawer.tsx | 21 +++++++++++++++++++ src/frontend/src/forms/ImporterForms.tsx | 0 src/frontend/src/pages/Index/Playground.tsx | 17 +++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 src/frontend/src/components/importer/ImporterDrawer.tsx create mode 100644 src/frontend/src/forms/ImporterForms.tsx diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx new file mode 100644 index 000000000000..23d7953eb204 --- /dev/null +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -0,0 +1,21 @@ +import { Drawer } from '@mantine/core'; + +export default function ImporterDrawer({ + opened, + onClose +}: { + opened: boolean; + onClose: () => void; +}) { + return ( + +
Hello world!
+ + ); +} diff --git a/src/frontend/src/forms/ImporterForms.tsx b/src/frontend/src/forms/ImporterForms.tsx new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/frontend/src/pages/Index/Playground.tsx b/src/frontend/src/pages/Index/Playground.tsx index 3bbf0f527dc8..125dc614d8df 100644 --- a/src/frontend/src/pages/Index/Playground.tsx +++ b/src/frontend/src/pages/Index/Playground.tsx @@ -5,6 +5,7 @@ import { Accordion } from '@mantine/core'; import { ReactNode, useMemo, useState } from 'react'; import { OptionsApiForm } from '../../components/forms/ApiForm'; +import ImporterDrawer from '../../components/importer/ImporterDrawer'; import { PlaceholderPill } from '../../components/items/Placeholder'; import { StylishText } from '../../components/items/StylishText'; import { StatusRenderer } from '../../components/render/StatusRenderer'; @@ -167,6 +168,18 @@ function StatusLabelPlayground() { ); } +// Data importing +function DataImportingPlayground() { + const [opened, setOpened] = useState(false); + + return ( + <> + + setOpened(false)} /> + + ); +} + /** Construct a simple accordion group with title and content */ function PlaygroundArea({ title, @@ -207,6 +220,10 @@ export default function Playground() { title="Status labels" content={} /> + } + /> ); From 9b66bb107ac75fe751c7069ab4a0e0e620e686ea Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 25 Mar 2024 12:40:50 +0000 Subject: [PATCH 040/190] Adds simple form to start new import session - Needs work, file field does not currently function correctly --- src/frontend/src/enums/ApiEndpoints.tsx | 6 ++++++ src/frontend/src/forms/ImporterForms.tsx | 20 ++++++++++++++++++ src/frontend/src/pages/Index/Playground.tsx | 23 ++++++++++++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/frontend/src/enums/ApiEndpoints.tsx b/src/frontend/src/enums/ApiEndpoints.tsx index f0910099da96..a87c704fe96b 100644 --- a/src/frontend/src/enums/ApiEndpoints.tsx +++ b/src/frontend/src/enums/ApiEndpoints.tsx @@ -44,6 +44,12 @@ export enum ApiEndpoints { group_list = 'user/group/', owner_list = 'user/owner/', + // Data import endpoints + import_session_list = 'importer/session/', + import_session_accept_fields = 'importer/session/:id/accept_fields/', + import_session_column_mapping_list = 'importer/column-mapping/', + import_session_row_list = 'importer/row/', + // Notification endpoints notifications_list = 'notifications/', notifications_readall = 'notifications/readall/', diff --git a/src/frontend/src/forms/ImporterForms.tsx b/src/frontend/src/forms/ImporterForms.tsx index e69de29bb2d1..498937cdfa61 100644 --- a/src/frontend/src/forms/ImporterForms.tsx +++ b/src/frontend/src/forms/ImporterForms.tsx @@ -0,0 +1,20 @@ +import { useMemo } from 'react'; + +import { ApiFormFieldSet } from '../components/forms/fields/ApiFormField'; + +export function dataImporterSessionFields({ model }: { model: string }) { + return useMemo(() => { + const fields: ApiFormFieldSet = { + data_file: {}, + model_type: { + value: model, + hidden: true + }, + field_detauls: { + hidden: true + } + }; + + return fields; + }, []); +} diff --git a/src/frontend/src/pages/Index/Playground.tsx b/src/frontend/src/pages/Index/Playground.tsx index 125dc614d8df..f5559dbeec37 100644 --- a/src/frontend/src/pages/Index/Playground.tsx +++ b/src/frontend/src/pages/Index/Playground.tsx @@ -11,6 +11,7 @@ import { StylishText } from '../../components/items/StylishText'; import { StatusRenderer } from '../../components/render/StatusRenderer'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { ModelType } from '../../enums/ModelType'; +import { dataImporterSessionFields } from '../../forms/ImporterForms'; import { partCategoryFields, usePartFields } from '../../forms/PartForms'; import { useCreateStockItem } from '../../forms/StockForms'; import { @@ -172,9 +173,29 @@ function StatusLabelPlayground() { function DataImportingPlayground() { const [opened, setOpened] = useState(false); + const importSessionFields = dataImporterSessionFields({ + model: 'partcategory' + }); + + const createNewImportSession = useCreateApiFormModal({ + url: ApiEndpoints.import_session_list, + title: 'Create Import Session', + fields: importSessionFields, + initialData: { + parent: 1 + }, + onFormSuccess: (response: any) => { + console.log('response:'); + console.log(response); + } + }); + return ( <> - + {createNewImportSession.modal} + setOpened(false)} /> ); From 1fa7a4a9f052490658132173434e062d3679cb92 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 22:05:19 +0000 Subject: [PATCH 041/190] data_file is *not* read_only --- InvenTree/importer/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/InvenTree/importer/serializers.py b/InvenTree/importer/serializers.py index 8f4f4fb5a1e6..e3353efc356b 100644 --- a/InvenTree/importer/serializers.py +++ b/InvenTree/importer/serializers.py @@ -46,7 +46,7 @@ class Meta: ] read_only_fields = ['pk', 'user', 'status', 'columns'] - data_file = InvenTreeAttachmentSerializerField(read_only=True) + data_file = InvenTreeAttachmentSerializerField() available_fields = serializers.JSONField(read_only=True) From 156def362e04116bea469f1d747610a3a1821092 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 22:09:58 +0000 Subject: [PATCH 042/190] Add check for file type --- InvenTree/importer/operations.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/InvenTree/importer/operations.py b/InvenTree/importer/operations.py index 0f53d182b06f..d1b7b2ff2b12 100644 --- a/InvenTree/importer/operations.py +++ b/InvenTree/importer/operations.py @@ -5,6 +5,8 @@ import tablib +import InvenTree.helpers + def load_data_file(data_file, format=None): """Load data file into a tablib dataset. @@ -20,6 +22,9 @@ def load_data_file(data_file, format=None): if format and format.startswith('.'): format = format[1:] + if format not in InvenTree.helpers.GetExportFormats(): + raise ValidationError(_('Unsupported data file format')) + file_object = data_file.file if hasattr(file_object, 'open'): From 33f4ba7c03867f4698b4bcfba275826b8e65d229 Mon Sep 17 00:00:00 2001 From: Oliver Date: Mon, 25 Mar 2024 22:11:13 +0000 Subject: [PATCH 043/190] Remove debug statements --- InvenTree/importer/models.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 4328a8864243..65282451aecc 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -408,10 +408,6 @@ def serializer_data(self): """ session_defaults = self.session.field_defaults or {} - print('serializer_data:') - print('- session_defaults:', session_defaults, ':', type(session_defaults)) - print('- self.data:', self.data, ':', type(self.data)) - return {**session_defaults, **self.data} def construct_serializer(self): From 1695ff3c410bf32f6d08ed33bff8030b55291848 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 26 Mar 2024 01:12:53 +0000 Subject: [PATCH 044/190] Refactor column mapping - Generate mapping for each column - Remove "columns" field - Column names are calculated dynamically --- InvenTree/importer/migrations/0001_initial.py | 1 - InvenTree/importer/models.py | 72 ++++++++++--------- 2 files changed, 40 insertions(+), 33 deletions(-) diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index b42d6d0e68cd..cbb3a8bd3e72 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -27,7 +27,6 @@ class Migration(migrations.Migration): ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=GetExportFormats()), importer.validators.validate_data_file], verbose_name='Data File')), ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), - ('columns', models.JSONField(blank=True, null=True, verbose_name='Columns')), ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults', validators=[importer.validators.validate_field_defaults])), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 65282451aecc..6d419838bf53 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -30,7 +30,6 @@ class DataImportSession(models.Model): data_file: FileField for the data file to import status: IntegerField for the status of the import session user: ForeignKey to the User who initiated the import - data_columns: JSONField for the data columns in the import file (mapped to database columns) field_defaults: JSONField for field overrides (e.g. custom field values) """ @@ -51,10 +50,7 @@ def save(self, *args, **kwargs): # New object - run initial setup self.status = DataImportStatusCode.INITIAL.value self.progress = 0 - self.extract_column_names() - self.auto_assign_columns() - self.save() - + self.create_columns() self.trigger_data_import() timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_('Timestamp')) @@ -71,15 +67,6 @@ def save(self, *args, **kwargs): ], ) - columns = models.JSONField(blank=True, null=True, verbose_name=_('Columns')) - - def extract_column_names(self): - """Extract column names from uploaded file. - - This method is called once, when the file is first uploaded. - """ - self.columns = importer.operations.extract_column_names(self.data_file) - model_type = models.CharField( blank=False, max_length=100, @@ -134,32 +121,53 @@ def serializer_fields(self, required=None, read_only=False): return get_fields(self.serializer, required=required, read_only=read_only) - def auto_assign_columns(self): - """Automatically assign columns based on the serializer fields.""" - available_columns = self.columns - serializer_fields = self.serializer_fields() + @property + def columns(self) -> list: + """Returns a list of the columns available for this data import session.""" + return [m.column for m in self.column_mappings.all()] + + def create_columns(self): + """Generate column mappings based on the serializer fields. + + This method is called once, when the file is first imported. + """ + # Extract list of column names from the file + columns = importer.operations.extract_column_names(self.data_file) + + serializer_fields = self.serializer_fields(read_only=False) # Remove any existing mappings self.column_mappings.all().delete() - for name, field in serializer_fields.items(): - column = None - label = getattr(field, 'label', name) + matched_fields = set() + + # Create a default mapping for each column in the dataset + for column in columns: + field_name = '' + + # Check each field for a direct match + for field, field_def in serializer_fields.items(): + # Ignore if this field has already been matched to a column + if field in matched_fields: + continue - if name in available_columns: - column = name - elif label in available_columns: - column = label + field_options = [field, getattr(field_def, 'label', field)] - if column is not None: - try: - DataImportColumnMap.objects.create( - session=self, field=name, column=column - ) - except Exception: - pass + if column in field_options: + field_name = field + break + + if column.lower() in [f.lower() for f in field_options]: + field_name = field + break + + # Generate a new mapping + DataImportColumnMap.objects.create( + session=self, column=column, field=field_name + ) self.status = DataImportStatusCode.MAPPING.value + self.save() def accept_mapping(self): """Accept current mapping configuration. From 3897a38cc3a7bee6bd4f4a01a1b90619010e96fa Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 26 Mar 2024 01:21:16 +0000 Subject: [PATCH 045/190] Fix uniqueness requirements on mapping table --- InvenTree/importer/migrations/0001_initial.py | 3 -- InvenTree/importer/models.py | 38 ++++++++++++++++--- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/InvenTree/importer/migrations/0001_initial.py b/InvenTree/importer/migrations/0001_initial.py index cbb3a8bd3e72..c1f20efd6526 100644 --- a/InvenTree/importer/migrations/0001_initial.py +++ b/InvenTree/importer/migrations/0001_initial.py @@ -39,9 +39,6 @@ class Migration(migrations.Migration): ('field', models.CharField(max_length=100, verbose_name='Field')), ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='column_mappings', to='importer.dataimportsession', verbose_name='Import Session')), ], - options={ - 'unique_together': {('session', 'column'), ('session', 'field')}, - }, ), migrations.CreateModel( name='DataImportRow', diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 6d419838bf53..3d529ddbd8c0 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -1,6 +1,7 @@ """Model definitions for the 'importer' app.""" import logging +from typing import Collection from django.contrib.auth.models import User from django.core.exceptions import ValidationError as DjangoValidationError @@ -141,6 +142,8 @@ def create_columns(self): matched_fields = set() + column_mappings = [] + # Create a default mapping for each column in the dataset for column in columns: field_name = '' @@ -155,17 +158,21 @@ def create_columns(self): if column in field_options: field_name = field + matched_fields.add(field) break if column.lower() in [f.lower() for f in field_options]: field_name = field + matched_fields.add(field) break - # Generate a new mapping - DataImportColumnMap.objects.create( - session=self, column=column, field=field_name + column_mappings.append( + DataImportColumnMap(session=self, column=column, field=field_name) ) + # Create the column mappings + DataImportColumnMap.objects.bulk_create(column_mappings) + self.status = DataImportStatusCode.MAPPING.value self.save() @@ -295,10 +302,25 @@ def get_api_url(): """Return the API URL associated with the DataImportColumnMap model.""" return reverse('api-importer-mapping-list') - class Meta: - """Model meta options.""" + def save(self, *args, **kwargs): + """Save the DataImportColumnMap object.""" + self.clean() + self.validate_unique() + + super().save(*args, **kwargs) + + def validate_unique(self, exclude=None): + """Ensure that the column mapping is unique within the session.""" + super().validate_unique(exclude) + + if self.session.column_mappings.filter(column=self.column).exists(): + raise DjangoValidationError({'column': _('Column is already mapped')}) - unique_together = [('session', 'column'), ('session', 'field')] + if ( + self.field not in ['', None] + and self.session.column_mappings.filter(field=self.field).exists() + ): + raise DjangoValidationError({'field': _('Field is already mapped')}) def clean(self): """Validate the column mapping.""" @@ -314,6 +336,10 @@ def clean(self): 'column': _('Column does not exist in the data file') }) + # Field is allowed to be empty + if self.field in ['', None]: + return + field_def = self.field_definition if not field_def: From 0eefb140d8f9ac602e132007d878e7b7b183f5a5 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 26 Mar 2024 01:23:56 +0000 Subject: [PATCH 046/190] Admin updates - Prevent deletion of mappings - Prevent addition of mappings --- InvenTree/importer/admin.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/InvenTree/importer/admin.py b/InvenTree/importer/admin.py index c18aa2037347..271cf004e48f 100644 --- a/InvenTree/importer/admin.py +++ b/InvenTree/importer/admin.py @@ -1,6 +1,9 @@ """Admin site specification for the 'importer' app.""" +from typing import Any + from django.contrib import admin +from django.http import HttpRequest import importer.models @@ -9,6 +12,12 @@ class DataImportColumnMapAdmin(admin.TabularInline): """Inline admin for DataImportColumnMap model.""" model = importer.models.DataImportColumnMap + can_delete = False + max_num = 0 + + def get_readonly_fields(self, request, obj=None): + """Return the readonly fields for the admin interface.""" + return ['column'] @admin.register(importer.models.DataImportSession) From 6cace47e40108f1bb61e1b8a84227c478db5800d Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 26 Mar 2024 01:33:19 +0000 Subject: [PATCH 047/190] API endpoint updates - Prevent mappings from being deleted - Prevent mappings from being created --- InvenTree/importer/admin.py | 1 - InvenTree/importer/api.py | 11 ++++++++--- InvenTree/importer/serializers.py | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/InvenTree/importer/admin.py b/InvenTree/importer/admin.py index 271cf004e48f..dfb3b02b3075 100644 --- a/InvenTree/importer/admin.py +++ b/InvenTree/importer/admin.py @@ -3,7 +3,6 @@ from typing import Any from django.contrib import admin -from django.http import HttpRequest import importer.models diff --git a/InvenTree/importer/api.py b/InvenTree/importer/api.py index 263e9625d40e..d1f8ae316c97 100644 --- a/InvenTree/importer/api.py +++ b/InvenTree/importer/api.py @@ -12,7 +12,12 @@ import importer.registry import importer.serializers from InvenTree.filters import SEARCH_ORDER_FILTER -from InvenTree.mixins import ListAPI, ListCreateAPI, RetrieveUpdateDestroyAPI +from InvenTree.mixins import ( + ListAPI, + ListCreateAPI, + RetrieveUpdateAPI, + RetrieveUpdateDestroyAPI, +) class DataImporterModelList(APIView): @@ -69,7 +74,7 @@ def post(self, request, pk): return Response(importer.serializers.DataImportSessionSerializer(session).data) -class DataImportColumnMappingList(ListCreateAPI): +class DataImportColumnMappingList(ListAPI): """API endpoint for accessing a list of DataImportColumnMap objects.""" queryset = importer.models.DataImportColumnMap.objects.all() @@ -80,7 +85,7 @@ class DataImportColumnMappingList(ListCreateAPI): filterset_fields = ['session'] -class DataImportColumnMappingDetail(RetrieveUpdateDestroyAPI): +class DataImportColumnMappingDetail(RetrieveUpdateAPI): """Detail endpoint for a single DataImportColumnMap object.""" queryset = importer.models.DataImportColumnMap.objects.all() diff --git a/InvenTree/importer/serializers.py b/InvenTree/importer/serializers.py index e3353efc356b..0feee9490374 100644 --- a/InvenTree/importer/serializers.py +++ b/InvenTree/importer/serializers.py @@ -19,6 +19,7 @@ class Meta: model = importer.models.DataImportColumnMap fields = ['pk', 'session', 'column', 'field', 'label'] + read_only_fields = ['column', 'session'] label = serializers.CharField(read_only=True) From e25eaa7592496379cb6f9f5fcc0358a54fdf819e Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 26 Mar 2024 01:59:45 +0000 Subject: [PATCH 048/190] Update importer drawer --- .../components/importer/ImporterDrawer.tsx | 48 ++++++++++++++++++- src/frontend/src/hooks/UseInstance.tsx | 2 +- src/frontend/src/pages/Index/Playground.tsx | 19 ++++++-- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index 23d7953eb204..ad2f1f195e0a 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -1,12 +1,48 @@ -import { Drawer } from '@mantine/core'; +import { Drawer, LoadingOverlay, Stack, Text } from '@mantine/core'; +import { useMemo } from 'react'; + +import { ApiEndpoints } from '../../enums/ApiEndpoints'; +import { useInstance } from '../../hooks/UseInstance'; export default function ImporterDrawer({ + sessionId, opened, onClose }: { + sessionId: number; opened: boolean; onClose: () => void; }) { + const { + instance: session, + refreshInstance, + instanceQuery + } = useInstance({ + endpoint: ApiEndpoints.import_session_list, + pk: sessionId, + refetchOnMount: true + }); + + // Construct list of field selections + const fieldOptions = useMemo(() => { + const fields = session?.available_fields ?? {}; + + let options = []; + + for (const key in fields) { + let field = fields[key]; + + if (!field.read_only) { + options.push({ + value: key, + label: field.label ?? key + }); + } + } + + return options; + }, [session]); + return ( -
Hello world!
+ + + Model Type: {session.model_type} + Rows: {session.row_count} + Columns: + {session?.column_mappings?.map((column: any) => ( + {column.column} + ))} + ); } diff --git a/src/frontend/src/hooks/UseInstance.tsx b/src/frontend/src/hooks/UseInstance.tsx index d160269d1472..02f2a2fa93f9 100644 --- a/src/frontend/src/hooks/UseInstance.tsx +++ b/src/frontend/src/hooks/UseInstance.tsx @@ -26,7 +26,7 @@ export function useInstance({ throwError = false }: { endpoint: ApiEndpoints; - pk?: string | undefined; + pk?: string | number | undefined; hasPrimaryKey?: boolean; params?: any; pathParams?: PathParams; diff --git a/src/frontend/src/pages/Index/Playground.tsx b/src/frontend/src/pages/Index/Playground.tsx index f5559dbeec37..1c889d37df23 100644 --- a/src/frontend/src/pages/Index/Playground.tsx +++ b/src/frontend/src/pages/Index/Playground.tsx @@ -2,7 +2,7 @@ import { Trans } from '@lingui/macro'; import { Button, Card, Stack, TextInput } from '@mantine/core'; import { Group, Text } from '@mantine/core'; import { Accordion } from '@mantine/core'; -import { ReactNode, useMemo, useState } from 'react'; +import { ReactNode, useCallback, useMemo, useState } from 'react'; import { OptionsApiForm } from '../../components/forms/ApiForm'; import ImporterDrawer from '../../components/importer/ImporterDrawer'; @@ -177,6 +177,8 @@ function DataImportingPlayground() { model: 'partcategory' }); + const [importSessionId, setImportSessionId] = useState(0); + const createNewImportSession = useCreateApiFormModal({ url: ApiEndpoints.import_session_list, title: 'Create Import Session', @@ -190,13 +192,20 @@ function DataImportingPlayground() { } }); + const openDrawer = useCallback(() => { + setImportSessionId(1); + setOpened(true); + }, []); + return ( <> {createNewImportSession.modal} - - setOpened(false)} /> + + setOpened(false)} + /> ); } From 7a0156713083ebcb77c74f59bc3f504533a669cb Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 26 Mar 2024 03:54:35 +0000 Subject: [PATCH 049/190] Add widget for selecting data columns --- InvenTree/importer/models.py | 1 - .../importer/ImporterColumnSelector.tsx | 43 ++++++++++++ .../components/importer/ImporterDrawer.tsx | 67 +++++++++++++------ .../src/components/render/StatusRenderer.tsx | 59 ++++++++++++---- src/frontend/src/defaults/backendMappings.tsx | 3 +- src/frontend/src/enums/ModelType.tsx | 1 + 6 files changed, 138 insertions(+), 36 deletions(-) create mode 100644 src/frontend/src/components/importer/ImporterColumnSelector.tsx diff --git a/InvenTree/importer/models.py b/InvenTree/importer/models.py index 3d529ddbd8c0..7e99c2bf9e7a 100644 --- a/InvenTree/importer/models.py +++ b/InvenTree/importer/models.py @@ -52,7 +52,6 @@ def save(self, *args, **kwargs): self.status = DataImportStatusCode.INITIAL.value self.progress = 0 self.create_columns() - self.trigger_data_import() timestamp = models.DateTimeField(auto_now_add=True, verbose_name=_('Timestamp')) diff --git a/src/frontend/src/components/importer/ImporterColumnSelector.tsx b/src/frontend/src/components/importer/ImporterColumnSelector.tsx new file mode 100644 index 000000000000..983cf72e4d5d --- /dev/null +++ b/src/frontend/src/components/importer/ImporterColumnSelector.tsx @@ -0,0 +1,43 @@ +import { t } from '@lingui/macro'; +import { Button, Group, Select, Stack, Text } from '@mantine/core'; +import { useMemo } from 'react'; + +export default function ImporterColumnSelector({ session }: { session: any }) { + // Available fields + const fields = useMemo(() => { + const available_fields = session?.available_fields ?? {}; + let options = []; + + for (const key in available_fields) { + let field = available_fields[key]; + + if (!field.read_only) { + options.push({ + value: key, + label: field.label ?? key + }); + } + } + + return options; + }, [session]); + + return ( + + + {t`Map data columns to database fields`} + + + {session?.column_mappings?.map((column: any) => { + return ( + + + ); +} export default function ImporterColumnSelector({ session }: { session: any }) { // Available fields @@ -32,11 +89,7 @@ export default function ImporterColumnSelector({ session }: { session: any }) { >{t`Accept Column Mapping`} {session?.column_mappings?.map((column: any) => { - return ( - - @@ -60,45 +69,39 @@ function ImporterColumn({ column, options }: { column: any; options: any }) { } export default function ImporterColumnSelector({ - session, - onComplete + session }: { - session: any; - onComplete: () => void; + session: ImportSessionState; }) { - // Available fields - const fields = useMemo(() => { - const available_fields = session?.available_fields ?? {}; - let options = []; - - for (const key in available_fields) { - let field = available_fields[key]; - - if (!field.read_only) { - options.push({ - value: key, - label: field.label ?? key - }); - } - } - - return options; - }, [session]); - const [errorMessage, setErrorMessage] = useState(''); const acceptMapping = useCallback(() => { - const url = apiUrl(ApiEndpoints.import_session_accept_fields, session.pk); + const url = apiUrl( + ApiEndpoints.import_session_accept_fields, + session.sessionId + ); api .post(url) - .then((response) => { - onComplete(); + .then(() => { + session.refreshSession(); }) .catch((error) => { setErrorMessage(error.response?.data?.error ?? t`An error occurred`); }); - }, [session]); + }, [session.sessionId]); + + const columnOptions: any[] = useMemo(() => { + return [ + { value: '', label: t`Ignore this field` }, + ...session.columnMappings.map((column: any) => { + return { + value: column.column, + label: column.column + }; + }) + ]; + }, [session.columnMappings]); return ( @@ -115,9 +118,23 @@ export default function ImporterColumnSelector({ {errorMessage} )} - {session?.column_mappings?.map((column: any) => { - return ; - })} + + {t`Database Field`} + {t`Imported Column Name`} + + + {session.columnMappings.map((column: any) => { + return [ + + {column.label ?? column.field} + + {column.description} + + , + + ]; + })} + ); } diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index 38f40c91e21f..9486b4df6f53 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -6,20 +6,17 @@ import { Group, LoadingOverlay, Paper, - Progress, ScrollArea, Stack, Text } from '@mantine/core'; import { useCallback, useMemo } from 'react'; -import { api } from '../../App'; -import { ApiEndpoints } from '../../enums/ApiEndpoints'; -import { ModelType } from '../../enums/ModelType'; -import { useInstance } from '../../hooks/UseInstance'; -import { apiUrl } from '../../states/ApiState'; +import { + ImportSessionStatus, + useImportSession +} from '../../hooks/UseImportSession'; import { StylishText } from '../items/StylishText'; -import { getStatusCodeName } from '../render/StatusRenderer'; import ImporterDataSelector from './ImportDataSelector'; import ImporterColumnSelector from './ImporterColumnSelector'; @@ -32,44 +29,23 @@ export default function ImporterDrawer({ opened: boolean; onClose: () => void; }) { - const { - instance: session, - refreshInstance, - instanceQuery - } = useInstance({ - endpoint: ApiEndpoints.import_session_list, - pk: sessionId, - refetchOnMount: true - }); - - const statusText = useMemo(() => { - const status = getStatusCodeName(ModelType.importsession, session?.status); + const session = useImportSession({ sessionId: sessionId }); - return status; - }, [session]); + const title: any = useMemo(() => { + return session.sessionData?.statusText ?? t`Importing Data`; + }, [session.sessionData]); - // TODO: This needs a lot of cleanup!! const widget = useMemo(() => { - switch (statusText) { - case 'INITIAL': - return Initial; - case 'MAPPING': - return ( - - ); - case 'IMPORTING': + switch (session.status) { + case ImportSessionStatus.INITIAL: + return Initial : TODO; + case ImportSessionStatus.MAPPING: + return ; + case ImportSessionStatus.IMPORTING: return Importing...; - case 'PROCESSING': - return ( - - ); - case 'COMPLETE': + case ImportSessionStatus.PROCESSING: + return ; + case ImportSessionStatus.COMPLETE: return Complete!; default: return Unknown status code: {session?.status}; @@ -78,7 +54,7 @@ export default function ImporterDrawer({ const cancelImport = useCallback(() => { // Cancel import session by deleting on the server - api.delete(apiUrl(ApiEndpoints.import_session_list, sessionId)); + session.cancelSession(); // Close the modal onClose(); @@ -97,12 +73,7 @@ export default function ImporterDrawer({ {t`Importing Data`} - + {title} @@ -82,7 +82,7 @@ export default function ImporterDrawer({ - + {/* TODO: Fix the header, while the content scrolls! */} {session.sessionQuery.isFetching || widget} From ed210bec3c15efbfd720930dc865661465da77bf Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 10 Jun 2024 12:06:35 +0000 Subject: [PATCH 105/190] spacing -> gap --- src/frontend/src/components/importer/ImportDataSelector.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 03c1a7ac9e9a..f65df853c34e 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -132,7 +132,7 @@ export default function ImporterDataSelector({ switchable: false, render: (row: any) => { return ( - + {row.row_index} {row.valid ? ( From c950c8f0b9ea406d948363b632560fc818ba51bb Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 10 Jun 2024 12:17:00 +0000 Subject: [PATCH 106/190] Add functionality to pre-process form data before upload --- src/backend/InvenTree/importer/serializers.py | 2 +- src/frontend/src/components/forms/ApiForm.tsx | 7 +++++++ .../src/components/importer/ImportDataSelector.tsx | 13 +++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/backend/InvenTree/importer/serializers.py b/src/backend/InvenTree/importer/serializers.py index a5d57c817579..ebea79599232 100644 --- a/src/backend/InvenTree/importer/serializers.py +++ b/src/backend/InvenTree/importer/serializers.py @@ -102,7 +102,7 @@ class Meta: 'pk', 'session', 'row_index', - 'data', + 'row_data', 'errors', 'valid', 'complete', diff --git a/src/frontend/src/components/forms/ApiForm.tsx b/src/frontend/src/components/forms/ApiForm.tsx index c9b660174eb7..0ea9075e45db 100644 --- a/src/frontend/src/components/forms/ApiForm.tsx +++ b/src/frontend/src/components/forms/ApiForm.tsx @@ -67,6 +67,7 @@ export interface ApiFormAction { * @param successMessage : Optional message to display on successful form submission * @param onFormSuccess : A callback function to call when the form is submitted successfully. * @param onFormError : A callback function to call when the form is submitted with errors. + * @param processFormData : A callback function to process the form data before submission * @param modelType : Define a model type for this form * @param follow : Boolean, follow the result of the form (if possible) * @param table : Table to update on success (if provided) @@ -91,6 +92,7 @@ export interface ApiFormProps { successMessage?: string; onFormSuccess?: (data: any) => void; onFormError?: () => void; + processFormData?: (data: any) => any; table?: TableState; modelType?: ModelType; follow?: boolean; @@ -386,6 +388,11 @@ export function ApiForm({ } }); + // Optionally pre-process the data before submitting it + if (props.processFormData) { + data = props.processFormData(data); + } + return api({ method: method, url: url, diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index f65df853c34e..b352e1ee8cf6 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -113,6 +113,15 @@ export default function ImporterDataSelector({ title: t`Edit Data`, fields: selectedFields, initialData: selectedRow.data, + processFormData: (data: any) => { + // Construct fields back into a single object + return { + data: { + ...selectedRow.data, + ...data + } + }; + }, onFormSuccess: (row: any) => table.updateRecord(row) }); @@ -135,9 +144,9 @@ export default function ImporterDataSelector({ {row.row_index} {row.valid ? ( - + ) : ( - + )} ); From 18acaf036da85838cfc8a7f66d09abd8e70f080c Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 20 Jun 2024 11:00:36 +0000 Subject: [PATCH 107/190] Remove old references to download_queryset --- src/backend/InvenTree/part/api.py | 9 --------- src/backend/InvenTree/stock/api.py | 13 ------------- 2 files changed, 22 deletions(-) diff --git a/src/backend/InvenTree/part/api.py b/src/backend/InvenTree/part/api.py index c7f58b663458..19374d944efb 100644 --- a/src/backend/InvenTree/part/api.py +++ b/src/backend/InvenTree/part/api.py @@ -1224,15 +1224,6 @@ class PartList(PartMixin, DataExportViewMixin, ListCreateAPI): filterset_class = PartFilter is_create = True - def download_queryset(self, queryset, export_format): - """Download the filtered queryset as a data file.""" - dataset = PartResource().export(queryset=queryset) - - filedata = dataset.export(export_format) - filename = f'InvenTree_Parts.{export_format}' - - return DownloadFile(filedata, filename) - def filter_queryset(self, queryset): """Perform custom filtering of the queryset.""" params = self.request.query_params diff --git a/src/backend/InvenTree/stock/api.py b/src/backend/InvenTree/stock/api.py index e5401ce2aab0..7cea3d6ecdcc 100644 --- a/src/backend/InvenTree/stock/api.py +++ b/src/backend/InvenTree/stock/api.py @@ -1066,19 +1066,6 @@ def create(self, request, *args, **kwargs): headers=self.get_success_headers(serializer.data), ) - def download_queryset(self, queryset, export_format): - """Download this queryset as a file. - - Uses the APIDownloadMixin mixin class - """ - dataset = StockItemResource().export(queryset=queryset) - - filedata = dataset.export(export_format) - - filename = f'InvenTree_StockItems_{InvenTree.helpers.current_date().strftime("%d-%b-%Y")}.{export_format}' - - return DownloadFile(filedata, filename) - def get_queryset(self, *args, **kwargs): """Annotate queryset before returning.""" queryset = super().get_queryset(*args, **kwargs) From 0077f29aa08e63c2bc10ffa2d32fcdef93109baf Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 20 Jun 2024 12:17:29 +0000 Subject: [PATCH 108/190] Improvements for data import drawer: - Pin title at top of drawer --- .../components/importer/ImporterDrawer.tsx | 56 +++++++++++-------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index 6cdfbd3045a3..7090f19cdd4f 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -1,5 +1,6 @@ import { t } from '@lingui/macro'; import { + ActionIcon, Button, Divider, Drawer, @@ -8,14 +9,17 @@ import { Paper, ScrollArea, Stack, - Text + Text, + Tooltip } from '@mantine/core'; -import { useCallback, useMemo } from 'react'; +import { IconCircleX } from '@tabler/icons-react'; +import { ReactNode, useCallback, useMemo } from 'react'; import { ImportSessionStatus, useImportSession } from '../../hooks/UseImportSession'; +import { ProgressBar } from '../items/ProgressBar'; import { StylishText } from '../items/StylishText'; import ImporterDataSelector from './ImportDataSelector'; import ImporterColumnSelector from './ImporterColumnSelector'; @@ -31,10 +35,6 @@ export default function ImporterDrawer({ }) { const session = useImportSession({ sessionId: sessionId }); - const title: any = useMemo(() => { - return session.sessionData?.statusText ?? t`Importing Data`; - }, [session.sessionData]); - const widget = useMemo(() => { switch (session.status) { case ImportSessionStatus.INITIAL: @@ -60,34 +60,44 @@ export default function ImporterDrawer({ onClose(); }, [session]); + const title: ReactNode = useMemo(() => { + return ( + + + + {session.sessionData?.statusText ?? t`Importing Data`} + + + + + + + + + + + ); + }, []); + return ( - - {t`Importing Data`} - {title} - - - - - - - - - {/* TODO: Fix the header, while the content scrolls! */} - {session.sessionQuery.isFetching || widget} - - + + {session.sessionQuery.isFetching || widget} ); From cfce2c6059dd8d04f1276a51a54e15dc37dc089d Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 20 Jun 2024 12:25:29 +0000 Subject: [PATCH 109/190] Further improvements --- .../components/importer/ImporterDrawer.tsx | 33 +++++++++++++++++-- src/frontend/src/hooks/UseImportSession.tsx | 1 + 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index 7090f19cdd4f..918e0b39ce96 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -35,6 +35,23 @@ export default function ImporterDrawer({ }) { const session = useImportSession({ sessionId: sessionId }); + const description: string = useMemo(() => { + switch (session.status) { + case ImportSessionStatus.INITIAL: + return t`Data File Upload`; + case ImportSessionStatus.MAPPING: + return t`Mapping Data Columns`; + case ImportSessionStatus.IMPORTING: + return t`Importing Data`; + case ImportSessionStatus.PROCESSING: + return t`Processing Data`; + case ImportSessionStatus.COMPLETE: + return t`Import Complete`; + default: + return t`Unknown Status` + ` - ${session.status}`; + } + }, [session]); + const widget = useMemo(() => { switch (session.status) { case ImportSessionStatus.INITIAL: @@ -63,11 +80,20 @@ export default function ImporterDrawer({ const title: ReactNode = useMemo(() => { return ( - + {session.sessionData?.statusText ?? t`Importing Data`} - + + {description} + + @@ -90,6 +116,9 @@ export default function ImporterDrawer({ closeOnEscape={false} closeOnClickOutside={false} styles={{ + header: { + width: '90%' + }, title: { width: '100%' } diff --git a/src/frontend/src/hooks/UseImportSession.tsx b/src/frontend/src/hooks/UseImportSession.tsx index ce7357a223e0..910ba9a16096 100644 --- a/src/frontend/src/hooks/UseImportSession.tsx +++ b/src/frontend/src/hooks/UseImportSession.tsx @@ -9,6 +9,7 @@ import { useInstance } from './UseInstance'; * Custom hook for managing the state of a data import session */ +// TODO: Load these values from the server? export enum ImportSessionStatus { INITIAL = 0, MAPPING = 10, From 1a00326c98b41e4ae1e3f1a02aad3d0077582c94 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 20 Jun 2024 12:38:54 +0000 Subject: [PATCH 110/190] Fix column selection input --- .../importer/ImporterColumnSelector.tsx | 6 +++--- src/frontend/src/hooks/UseImportSession.tsx | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/frontend/src/components/importer/ImporterColumnSelector.tsx b/src/frontend/src/components/importer/ImporterColumnSelector.tsx index 3b04ef0e0b95..f765b9f73a86 100644 --- a/src/frontend/src/components/importer/ImporterColumnSelector.tsx +++ b/src/frontend/src/components/importer/ImporterColumnSelector.tsx @@ -89,10 +89,10 @@ export default function ImporterColumnSelector({ const columnOptions: any[] = useMemo(() => { return [ { value: '', label: t`Ignore this field` }, - ...session.columnMappings.map((column: any) => { + ...session.availableColumns.map((column: any) => { return { - value: column.column, - label: column.column + value: column, + label: column }; }) ]; diff --git a/src/frontend/src/hooks/UseImportSession.tsx b/src/frontend/src/hooks/UseImportSession.tsx index 910ba9a16096..09d8dcca8336 100644 --- a/src/frontend/src/hooks/UseImportSession.tsx +++ b/src/frontend/src/hooks/UseImportSession.tsx @@ -26,6 +26,7 @@ export type ImportSessionState = { sessionQuery: any; status: ImportSessionStatus; availableFields: Record; + availableColumns: string[]; mappedFields: any[]; columnMappings: any[]; }; @@ -61,6 +62,19 @@ export function useImportSession({ return sessionData?.available_fields ?? []; }, [sessionData]); + // List of available data file columns + const availableColumns: string[] = useMemo(() => { + let cols = sessionData?.columns ?? []; + + // Filter out any blank or duplicate columns + cols = cols.filter((col: string) => !!col); + cols = cols.filter( + (col: string, index: number) => cols.indexOf(col) === index + ); + + return cols; + }, [sessionData]); + const columnMappings: any[] = useMemo(() => { return sessionData?.column_mappings ?? []; }, [sessionData]); @@ -81,6 +95,7 @@ export function useImportSession({ sessionQuery, status, availableFields, + availableColumns, columnMappings, mappedFields }; From 60e728aee4142402ebf04256d5734817e3577dfa Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 20 Jun 2024 12:55:05 +0000 Subject: [PATCH 111/190] Formatting improvements --- .../importer/ImporterColumnSelector.tsx | 23 +++++++++++++------ src/frontend/src/hooks/UseImportSession.tsx | 10 ++++++-- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/frontend/src/components/importer/ImporterColumnSelector.tsx b/src/frontend/src/components/importer/ImporterColumnSelector.tsx index f765b9f73a86..e8c176ab7106 100644 --- a/src/frontend/src/components/importer/ImporterColumnSelector.tsx +++ b/src/frontend/src/components/importer/ImporterColumnSelector.tsx @@ -55,7 +55,7 @@ function ImporterColumn({ column, options }: { column: any; options: any[] }) { error={errorMessage} clearable placeholder={t`Select column, or leave blank to ignore this field.`} - label={column.column} + label={undefined} data={options} value={selectedColumn} onChange={onChange} @@ -113,19 +113,28 @@ export default function ImporterColumnSelector({ {errorMessage} )} - + {t`Database Field`} + {t`Field Description`} {t`Imported Column Name`} + {session.columnMappings.map((column: any) => { return [ - - {column.label ?? column.field} - - {column.description} + + + {column.label ?? column.field} - , + {column.required && ( + + * + + )} + , + + {column.description} + , ]; })} diff --git a/src/frontend/src/hooks/UseImportSession.tsx b/src/frontend/src/hooks/UseImportSession.tsx index 09d8dcca8336..2398c7407453 100644 --- a/src/frontend/src/hooks/UseImportSession.tsx +++ b/src/frontend/src/hooks/UseImportSession.tsx @@ -76,8 +76,14 @@ export function useImportSession({ }, [sessionData]); const columnMappings: any[] = useMemo(() => { - return sessionData?.column_mappings ?? []; - }, [sessionData]); + let mapping = + sessionData?.column_mappings?.map((mapping: any) => ({ + ...mapping, + ...(availableFields[mapping.field] ?? {}) + })) ?? []; + + return mapping; + }, [sessionData, availableColumns]); // List of field which have been mapped to columns const mappedFields: any[] = useMemo(() => { From edd49a0c21e726caf7952afb69ce8da0f0a54e8d Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 20 Jun 2024 13:05:02 +0000 Subject: [PATCH 112/190] Use a component for better progress display --- .../components/importer/ImporterDrawer.tsx | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index 918e0b39ce96..3902932477f1 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -1,29 +1,51 @@ import { t } from '@lingui/macro'; import { ActionIcon, - Button, Divider, Drawer, Group, LoadingOverlay, Paper, - ScrollArea, Stack, + Stepper, Text, Tooltip } from '@mantine/core'; import { IconCircleX } from '@tabler/icons-react'; -import { ReactNode, useCallback, useMemo } from 'react'; +import { ReactNode, useCallback, useMemo, useState } from 'react'; import { ImportSessionStatus, useImportSession } from '../../hooks/UseImportSession'; -import { ProgressBar } from '../items/ProgressBar'; import { StylishText } from '../items/StylishText'; import ImporterDataSelector from './ImportDataSelector'; import ImporterColumnSelector from './ImporterColumnSelector'; +/* + * Stepper component showing the current step of the data import process. + */ +function ImportDrawerStepper({ currentStep }: { currentStep: number }) { + /* TODO: Enhance this with: + * - Custom icons + * - Loading indicators for "background" states + */ + + return ( + + + + + + + ); +} + export default function ImporterDrawer({ sessionId, opened, @@ -35,6 +57,8 @@ export default function ImporterDrawer({ }) { const session = useImportSession({ sessionId: sessionId }); + const [currentStep, setCurrentStep] = useState(1); + const description: string = useMemo(() => { switch (session.status) { case ImportSessionStatus.INITIAL: @@ -90,10 +114,7 @@ export default function ImporterDrawer({ {session.sessionData?.statusText ?? t`Importing Data`} - - {description} - - + From 792cab6982cffdafee912b8dfea6bd020a171c6b Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 20 Jun 2024 13:07:33 +0000 Subject: [PATCH 113/190] Cleanup text --- src/frontend/src/components/importer/ImporterColumnSelector.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/components/importer/ImporterColumnSelector.tsx b/src/frontend/src/components/importer/ImporterColumnSelector.tsx index e8c176ab7106..b64333c875d6 100644 --- a/src/frontend/src/components/importer/ImporterColumnSelector.tsx +++ b/src/frontend/src/components/importer/ImporterColumnSelector.tsx @@ -88,7 +88,7 @@ export default function ImporterColumnSelector({ const columnOptions: any[] = useMemo(() => { return [ - { value: '', label: t`Ignore this field` }, + { value: '', label: t`Select a column from the data file` }, ...session.availableColumns.map((column: any) => { return { value: column, From f05197385054f2360ff7cb62de4d6acb8a4d025a Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sat, 22 Jun 2024 02:21:24 +0000 Subject: [PATCH 114/190] Add export-only fields to BuildItem queryset --- src/backend/InvenTree/build/api.py | 4 ++++ src/backend/InvenTree/build/serializers.py | 14 +++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/backend/InvenTree/build/api.py b/src/backend/InvenTree/build/api.py index b685567c8078..f645986106f4 100644 --- a/src/backend/InvenTree/build/api.py +++ b/src/backend/InvenTree/build/api.py @@ -580,6 +580,10 @@ def get_queryset(self): 'stock_item', 'stock_item__location', 'stock_item__part', + 'stock_item__supplier_part', + 'stock_item__supplier_part__manufacturer_part', + ).prefetch_related( + 'stock_item__location__tags', ) return queryset diff --git a/src/backend/InvenTree/build/serializers.py b/src/backend/InvenTree/build/serializers.py index f84d2ef4664c..05dfee894ded 100644 --- a/src/backend/InvenTree/build/serializers.py +++ b/src/backend/InvenTree/build/serializers.py @@ -1051,7 +1051,13 @@ def save(self): class BuildItemSerializer(DataImportExportSerializerMixin, InvenTreeModelSerializer): - """Serializes a BuildItem object.""" + """Serializes a BuildItem object, which is an allocation of a stock item against a build order.""" + + # These fields are only used for data export + export_only_fields = [ + 'sku', + 'mpn', + ] class Meta: """Serializer metaclass""" @@ -1067,8 +1073,14 @@ class Meta: 'part_detail', 'stock_item_detail', 'build_detail', + 'sku', + 'mpn', ] + # Export-only fields + sku = serializers.CharField(source='stock_item.supplier_part.SKU', label=_('Supplier Part Number'), read_only=True) + mpn = serializers.CharField(source='stock_item.supplier_part.manufacturer_part.MPN', label=_('Manufacturer Part Number'), read_only=True) + # Annotated fields build = serializers.PrimaryKeyRelatedField(source='build_line.build', many=False, read_only=True) From c42fd0e220dfc1eaf895b98302f965da8277c754 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sat, 22 Jun 2024 02:29:23 +0000 Subject: [PATCH 115/190] Expand "export" fields for BuildItem dataset --- src/backend/InvenTree/build/api.py | 1 + src/backend/InvenTree/build/serializers.py | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/backend/InvenTree/build/api.py b/src/backend/InvenTree/build/api.py index f645986106f4..8944ecb9a720 100644 --- a/src/backend/InvenTree/build/api.py +++ b/src/backend/InvenTree/build/api.py @@ -576,6 +576,7 @@ def get_queryset(self): queryset = queryset.select_related( 'build_line', 'build_line__build', + 'build_line__bom_item', 'install_into', 'stock_item', 'stock_item__location', diff --git a/src/backend/InvenTree/build/serializers.py b/src/backend/InvenTree/build/serializers.py index 05dfee894ded..db858877b480 100644 --- a/src/backend/InvenTree/build/serializers.py +++ b/src/backend/InvenTree/build/serializers.py @@ -1055,8 +1055,11 @@ class BuildItemSerializer(DataImportExportSerializerMixin, InvenTreeModelSeriali # These fields are only used for data export export_only_fields = [ + 'build_reference', + 'bom_reference', 'sku', 'mpn', + 'location_name', ] class Meta: @@ -1069,17 +1072,28 @@ class Meta: 'install_into', 'stock_item', 'quantity', + 'location', + + # Detail fields, can be included or excluded + 'build_detail', 'location_detail', 'part_detail', 'stock_item_detail', - 'build_detail', - 'sku', + + # The following fields are only used for data export + 'bom_reference', + 'build_reference', + 'location_name', 'mpn', + 'sku', ] # Export-only fields sku = serializers.CharField(source='stock_item.supplier_part.SKU', label=_('Supplier Part Number'), read_only=True) mpn = serializers.CharField(source='stock_item.supplier_part.manufacturer_part.MPN', label=_('Manufacturer Part Number'), read_only=True) + location_name = serializers.CharField(source='stock_item.location.name', label=_('Location Name'), read_only=True) + build_reference = serializers.CharField(source='build.reference', label=_('Build Reference'), read_only=True) + bom_reference = serializers.CharField(source='build_line.bom_item.reference', label=_('BOM Reference'), read_only=True) # Annotated fields build = serializers.PrimaryKeyRelatedField(source='build_line.build', many=False, read_only=True) @@ -1087,6 +1101,7 @@ class Meta: # Extra (optional) detail fields part_detail = PartBriefSerializer(source='stock_item.part', many=False, read_only=True, pricing=False) stock_item_detail = StockItemSerializerBrief(source='stock_item', read_only=True) + location = serializers.PrimaryKeyRelatedField(source='stock_item.location', many=False, read_only=True) location_detail = LocationSerializer(source='stock_item.location', read_only=True) build_detail = BuildSerializer(source='build_line.build', many=False, read_only=True) From 922ed09a1b74aa940eeeb84cbc7208a1b64ba743 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sat, 22 Jun 2024 02:38:12 +0000 Subject: [PATCH 116/190] Skip backup and static steps in CI --- .github/actions/setup/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup/action.yaml b/.github/actions/setup/action.yaml index 41e5db55a9e0..cbe9ac3e5ca6 100644 --- a/.github/actions/setup/action.yaml +++ b/.github/actions/setup/action.yaml @@ -93,4 +93,4 @@ runs: - name: Run invoke update if: ${{ inputs.update == 'true' }} shell: bash - run: invoke update --uv + run: invoke update --uv --skip-backup --skip-static From 66249da32655fd178a302ecb19d79e76274fa2dc Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sat, 22 Jun 2024 07:53:44 +0000 Subject: [PATCH 117/190] Remove hard-coded paths --- src/backend/InvenTree/InvenTree/settings.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/settings.py b/src/backend/InvenTree/InvenTree/settings.py index 018ef5dc6143..ecc1e4ca9d6f 100644 --- a/src/backend/InvenTree/InvenTree/settings.py +++ b/src/backend/InvenTree/InvenTree/settings.py @@ -573,7 +573,6 @@ """ db_engine = db_config['ENGINE'].lower() -db_engine = 'sqlite' # Correct common misspelling if db_engine == 'sqlite': @@ -587,8 +586,6 @@ db_name = db_config['NAME'] db_host = db_config.get('HOST', "''") -db_name = '/home/inventree/inventree-db.sqlite3' - if 'sqlite' in db_engine: db_name = str(Path(db_name).resolve()) db_config['NAME'] = db_name From be504899eb6b1b65fe6320f9a60ed0f234a3ac6f Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 24 Jun 2024 12:50:35 +0000 Subject: [PATCH 118/190] Fix for "accept_mapping" method --- src/backend/InvenTree/importer/models.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 65a48b1bf8d0..51aeafcf590f 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -193,14 +193,15 @@ def accept_mapping(self): missing_fields = [] for field in required_fields: - # An explicit mapping exists - if self.column_mappings.filter(field=field).exists(): - continue - # A default value exists if field in field_defaults: continue + # The field has been mapped to a data column + if mapping := self.column_mappings.filter(field=field).first(): + if mapping.column: + continue + missing_fields.append(field) if len(missing_fields) > 0: From 138e1214b2fe882bb59b0069c14430f42b573cae Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 24 Jun 2024 13:12:36 +0000 Subject: [PATCH 119/190] Present required fields first on import session --- src/backend/InvenTree/InvenTree/metadata.py | 9 +++++++-- src/backend/InvenTree/importer/models.py | 2 -- src/backend/InvenTree/importer/views.py | 5 ----- src/frontend/src/hooks/UseImportSession.tsx | 6 ++++++ 4 files changed, 13 insertions(+), 9 deletions(-) delete mode 100644 src/backend/InvenTree/importer/views.py diff --git a/src/backend/InvenTree/InvenTree/metadata.py b/src/backend/InvenTree/InvenTree/metadata.py index ef203ddcfe37..3eaec8e17122 100644 --- a/src/backend/InvenTree/InvenTree/metadata.py +++ b/src/backend/InvenTree/InvenTree/metadata.py @@ -126,10 +126,10 @@ def override_value(self, field_name, field_value, model_value): - model_value is callable, and field_value is not (this indicates that the model value is translated) - model_value is not a string, and field_value is a string (this indicates that the model value is translated) """ - if model_value and not field_value: + if field_value is None and model_value is not None: return model_value - if field_value and not model_value: + if model_value is None and field_value is not None: return field_value if callable(model_value) and not callable(field_value): @@ -200,6 +200,11 @@ def get_serializer_info(self, serializer): field_value = serializer_info[name].get(field_key, None) model_value = getattr(field, model_key, None) + if name == 'category': + print('Category field:', field_key, model_key) + print(' - field_value:', field_value) + print(' - model_value:', model_value) + if value := self.override_value(name, field_value, model_value): serializer_info[name][field_key] = value diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 51aeafcf590f..6bc6aa2768e5 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -228,8 +228,6 @@ def trigger_data_import(self): def import_data(self): """Perform the data import process for this session.""" - # TODO: Clear any existing error messages - # Clear any existing data rows self.rows.all().delete() diff --git a/src/backend/InvenTree/importer/views.py b/src/backend/InvenTree/importer/views.py deleted file mode 100644 index 0f2e2926adcf..000000000000 --- a/src/backend/InvenTree/importer/views.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Viewsets for the 'importer' app.""" - -from django.shortcuts import render - -# Create your views here. diff --git a/src/frontend/src/hooks/UseImportSession.tsx b/src/frontend/src/hooks/UseImportSession.tsx index 2398c7407453..d28979079439 100644 --- a/src/frontend/src/hooks/UseImportSession.tsx +++ b/src/frontend/src/hooks/UseImportSession.tsx @@ -82,6 +82,12 @@ export function useImportSession({ ...(availableFields[mapping.field] ?? {}) })) ?? []; + mapping = mapping.sort((a: any, b: any) => { + if (a?.required && !b?.required) return -1; + if (!a?.required && b?.required) return 1; + return 0; + }); + return mapping; }, [sessionData, availableColumns]); From 2256b7c69e78f79994d9d75078e1d87bd24fd6a7 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 24 Jun 2024 13:50:27 +0000 Subject: [PATCH 120/190] Add "get_importable_fields" method --- src/backend/InvenTree/importer/mixins.py | 23 ++++++++++++++++++++++- src/backend/InvenTree/importer/models.py | 8 ++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/backend/InvenTree/importer/mixins.py b/src/backend/InvenTree/importer/mixins.py index e90f43398cdd..ea1a25a30aa7 100644 --- a/src/backend/InvenTree/importer/mixins.py +++ b/src/backend/InvenTree/importer/mixins.py @@ -43,6 +43,27 @@ def __init__(self, *args, **kwargs): for field in self.get_import_only_fields(**kwargs): self.fields.pop(field, None) + def get_importable_fields(self) -> dict: + """Return a dict of fields which can be imported against this serializer instance. + + Returns: + dict: A dictionary of field names and field objects + """ + fields = {} + + for name, field in self.fields.items(): + # Skip read-only fields + if getattr(field, 'read_only', False): + continue + + # Skip fields which are themselves serializers + if issubclass(field.__class__, serializers.Serializer): + continue + + fields[name] = field + + return fields + class DataExportSerializerMixin: """Mixin class for adding data export functionality to a DRF serializer.""" @@ -78,7 +99,7 @@ def __init__(self, *args, **kwargs): self.fields.pop(field, None) def get_exportable_fields(self) -> dict: - """Return a list of fields which can be exported. + """Return a dict of fields which can be exported against this serializer instance. Note: Any fields which should be excluded from export have already been removed diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 6bc6aa2768e5..59c20da2b14b 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -107,7 +107,7 @@ def field_mapping(self): return mapping @property - def serializer(self): + def serializer_class(self): """Return the serializer class for this importer.""" from importer.registry import supported_models @@ -122,7 +122,7 @@ def serializer_fields(self, required=None, read_only=False, write_only=None): from importer.operations import get_fields return get_fields( - self.serializer, + self.serializer_class, required=required, read_only=read_only, write_only=write_only, @@ -286,8 +286,8 @@ def available_fields(self): from InvenTree.metadata import InvenTreeMetadata metadata = InvenTreeMetadata() - if serializer := self.serializer: - fields = metadata.get_serializer_info(serializer(data={})) + if serializer := self.serializer_class: + fields = metadata.get_serializer_info(serializer(data={}, importing=True)) else: fields = {} From b65f20732c4fb2420cf7169f216173fc788e4458 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 24 Jun 2024 14:00:21 +0000 Subject: [PATCH 121/190] Add method for commiting imported row to database --- src/backend/InvenTree/importer/models.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 59c20da2b14b..97335bd165aa 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -248,7 +248,11 @@ def import_data(self): row_objects.append( importer.models.DataImportRow( - session=self, row_data=row_data, row_index=idx + session=self, + row_data=row_data, + row_index=idx, + valid=False, + complete=False, ) ) @@ -411,6 +415,7 @@ def get_api_url(): def save(self, *args, **kwargs): """Save the DataImportRow object.""" self.valid = self.validate() + self.complete = self.complete or False super().save(*args, **kwargs) session = models.ForeignKey( @@ -466,15 +471,22 @@ def construct_serializer(self): return serializer_class(data=self.serializer_data()) - def validate(self) -> bool: + def validate(self, commit=False) -> bool: """Validate the data in this row against the linked serializer. + Arguments: + commit: If True, the data is saved to the database (if validation passes) + Returns: True if the data is valid, False otherwise Raises: ValidationError: If the linked serializer is not valid """ + if self.complete: + # Row has already been completed + return True + serializer = self.construct_serializer() if not serializer: @@ -493,4 +505,11 @@ def validate(self) -> bool: if result: self.errors = None + if commit: + try: + serializer.save() + except Exception as e: + self.errors = {'non_field_errors': str(e)} + result = False + return result From bc812375a47978efdf7aa866ef9d8a89fff549ee Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 24 Jun 2024 14:00:43 +0000 Subject: [PATCH 122/190] Cleanup --- src/backend/InvenTree/importer/models.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 97335bd165aa..e8a718a73bac 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -248,11 +248,7 @@ def import_data(self): row_objects.append( importer.models.DataImportRow( - session=self, - row_data=row_data, - row_index=idx, - valid=False, - complete=False, + session=self, row_data=row_data, row_index=idx ) ) @@ -415,7 +411,6 @@ def get_api_url(): def save(self, *args, **kwargs): """Save the DataImportRow object.""" self.valid = self.validate() - self.complete = self.complete or False super().save(*args, **kwargs) session = models.ForeignKey( From 6e636e2e62a120d746257e2af49ba7fe2aa1713c Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 24 Jun 2024 14:06:48 +0000 Subject: [PATCH 123/190] Save "complete" state after row import --- src/backend/InvenTree/importer/models.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index e8a718a73bac..6b1e203ea654 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -459,12 +459,8 @@ def serializer_data(self): def construct_serializer(self): """Construct a serializer object for this row.""" - serializer_class = self.session.serializer - - if not serializer_class: - return None - - return serializer_class(data=self.serializer_data()) + if serializer_class := self.session.serializer_class: + return serializer_class(data=self.serializer_data()) def validate(self, commit=False) -> bool: """Validate the data in this row against the linked serializer. @@ -503,6 +499,8 @@ def validate(self, commit=False) -> bool: if commit: try: serializer.save() + self.complete = True + self.save() except Exception as e: self.errors = {'non_field_errors': str(e)} result = False From 2b43e65748e3a94c4f95497e37530f4df44442fd Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Mon, 24 Jun 2024 14:22:59 +0000 Subject: [PATCH 124/190] Allow prevention of column caching --- .../src/components/importer/ImportDataSelector.tsx | 6 ++++-- .../src/components/importer/ImporterDrawer.tsx | 2 +- src/frontend/src/tables/InvenTreeTable.tsx | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index b352e1ee8cf6..5de11ca4fb75 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -74,7 +74,7 @@ export default function ImporterDataSelector({ }: { session: ImportSessionState; }) { - const table = useTable('data-importer'); + const table = useTable('dataimporter'); const [selectedFieldNames, setSelectedFieldNames] = useState([]); @@ -229,7 +229,9 @@ export default function ImporterDataSelector({ }, rowActions: rowActions, tableFilters: filters, - enableColumnSwitching: true + enableColumnSwitching: true, + enableColumnCaching: false, + enableSelection: true }} /> diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index 3902932477f1..281ee335d266 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -91,7 +91,7 @@ export default function ImporterDrawer({ default: return Unknown status code: {session?.status}; } - }, [session]); + }, [session.status]); const cancelImport = useCallback(() => { // Cancel import session by deleting on the server diff --git a/src/frontend/src/tables/InvenTreeTable.tsx b/src/frontend/src/tables/InvenTreeTable.tsx index f52afa8a99cb..6e97c08fb2f8 100644 --- a/src/frontend/src/tables/InvenTreeTable.tsx +++ b/src/frontend/src/tables/InvenTreeTable.tsx @@ -74,6 +74,7 @@ const defaultPageSize: number = 25; * @param enablePagination : boolean - Enable pagination * @param enableRefresh : boolean - Enable refresh actions * @param enableColumnSwitching : boolean - Enable column switching + * @param enableColumnCaching : boolean - Enable caching of column names via API * @param pageSize : number - Number of records per page * @param barcodeActions : any[] - List of barcode actions * @param tableFilters : TableFilter[] - List of custom filters @@ -96,6 +97,7 @@ export type InvenTreeTableProps = { enablePagination?: boolean; enableRefresh?: boolean; enableColumnSwitching?: boolean; + enableColumnCaching?: boolean; enableLabels?: boolean; enableReports?: boolean; pageSize?: number; @@ -169,11 +171,14 @@ export function InvenTreeTable({ // Request OPTIONS data from the API, before we load the table const tableOptionQuery = useQuery({ enabled: true, - queryKey: ['options', url, tableState.tableKey], + queryKey: ['options', url, tableState.tableKey, props.enableColumnCaching], retry: 3, refetchOnMount: true, refetchOnWindowFocus: false, queryFn: async () => { + if (props.enableColumnCaching == false) { + return null; + } return api .options(url, { params: tableProps.params @@ -206,6 +211,10 @@ export function InvenTreeTable({ // Rebuild set of translated column names useEffect(() => { + if (props.enableColumnCaching == false) { + return; + } + const cacheKey = tableState.tableKey.split('-')[0]; // First check the local cache @@ -218,8 +227,9 @@ export function InvenTreeTable({ } // Otherwise, fetch the data from the API + console.log('refacthing table options:'); tableOptionQuery.refetch(); - }, [url, tableState.tableKey, props.params]); + }, [url, tableState.tableKey, props.params, props.enableColumnCaching]); // Build table properties based on provided props (and default props) const tableProps: InvenTreeTableProps = useMemo(() => { From ff494745bdf518a76237a7a75b9ccc3a29b1006e Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 06:38:13 +0000 Subject: [PATCH 125/190] Remove debug statement --- src/frontend/src/tables/InvenTreeTable.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/frontend/src/tables/InvenTreeTable.tsx b/src/frontend/src/tables/InvenTreeTable.tsx index 6e97c08fb2f8..202648d31b8d 100644 --- a/src/frontend/src/tables/InvenTreeTable.tsx +++ b/src/frontend/src/tables/InvenTreeTable.tsx @@ -227,7 +227,6 @@ export function InvenTreeTable({ } // Otherwise, fetch the data from the API - console.log('refacthing table options:'); tableOptionQuery.refetch(); }, [url, tableState.tableKey, props.params, props.enableColumnCaching]); From 4b632605cc185ac7004acac86f6a57f9e4fff685 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 09:37:14 +0000 Subject: [PATCH 126/190] Add basic admin table for import sessions --- src/backend/InvenTree/importer/api.py | 6 ++ src/backend/InvenTree/importer/serializers.py | 4 + .../InvenTree/importer/status_codes.py | 2 +- src/frontend/src/hooks/UseTable.tsx | 2 +- .../Index/Settings/AdminCenter/Index.tsx | 11 +++ .../tables/settings/ImportSessionTable.tsx | 80 +++++++++++++++++++ .../src/tables/settings/ProjectCodeTable.tsx | 8 +- 7 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 src/frontend/src/tables/settings/ImportSessionTable.tsx diff --git a/src/backend/InvenTree/importer/api.py b/src/backend/InvenTree/importer/api.py index 306b8ea364e0..7f6377a0c62b 100644 --- a/src/backend/InvenTree/importer/api.py +++ b/src/backend/InvenTree/importer/api.py @@ -48,6 +48,12 @@ class DataImportSessionList(ListCreateAPI): queryset = importer.models.DataImportSession.objects.all() serializer_class = importer.serializers.DataImportSessionSerializer + filter_backends = SEARCH_ORDER_FILTER + + filterset_fields = ['model_type', 'status'] + + ordering_fields = ['timestamp', 'status', 'model_type'] + class DataImportSessionDetail(RetrieveUpdateDestroyAPI): """Detail endpoint for a single DataImportSession object.""" diff --git a/src/backend/InvenTree/importer/serializers.py b/src/backend/InvenTree/importer/serializers.py index ebea79599232..45449f54352a 100644 --- a/src/backend/InvenTree/importer/serializers.py +++ b/src/backend/InvenTree/importer/serializers.py @@ -9,6 +9,7 @@ from InvenTree.serializers import ( InvenTreeAttachmentSerializerField, InvenTreeModelSerializer, + UserSerializer, ) @@ -41,6 +42,7 @@ class Meta: 'available_fields', 'status', 'user', + 'user_detail', 'columns', 'column_mappings', 'field_defaults', @@ -64,6 +66,8 @@ class Meta: column_mappings = DataImportColumnMapSerializer(many=True, read_only=True) + user_detail = UserSerializer(source='user', read_only=True, many=False) + def create(self, validated_data): """Override create method for this serializer. diff --git a/src/backend/InvenTree/importer/status_codes.py b/src/backend/InvenTree/importer/status_codes.py index d587ee68aeb0..71d4dfd0e69a 100644 --- a/src/backend/InvenTree/importer/status_codes.py +++ b/src/backend/InvenTree/importer/status_codes.py @@ -8,7 +8,7 @@ class DataImportStatusCode(StatusCode): """Defines a set of status codes for a DataImportSession.""" - INITIAL = 0, _('Initial'), 'secondary' # Import session has been created + INITIAL = 0, _('Initializing'), 'secondary' # Import session has been created MAPPING = 10, _('Mapping Columns'), 'primary' # Import fields are being mapped IMPORTING = 20, _('Importing Data'), 'primary' # Data is being imported PROCESSING = ( diff --git a/src/frontend/src/hooks/UseTable.tsx b/src/frontend/src/hooks/UseTable.tsx index ff9362291f70..b1bf2ce73574 100644 --- a/src/frontend/src/hooks/UseTable.tsx +++ b/src/frontend/src/hooks/UseTable.tsx @@ -58,7 +58,7 @@ export function useTable(tableName: string): TableState { // Callback used to refresh (reload) the table const refreshTable = useCallback(() => { setTableKey(generateTableName()); - }, []); + }, [generateTableName]); // Array of active filters (saved to local storage) const [activeFilters, setActiveFilters] = useLocalStorage({ diff --git a/src/frontend/src/pages/Index/Settings/AdminCenter/Index.tsx b/src/frontend/src/pages/Index/Settings/AdminCenter/Index.tsx index a5d16d530a27..da0a71e8a2b3 100644 --- a/src/frontend/src/pages/Index/Settings/AdminCenter/Index.tsx +++ b/src/frontend/src/pages/Index/Settings/AdminCenter/Index.tsx @@ -5,6 +5,7 @@ import { IconCpu, IconDevicesPc, IconExclamationCircle, + IconFileUpload, IconList, IconListDetails, IconPackages, @@ -49,6 +50,10 @@ const ErrorReportTable = Loadable( lazy(() => import('../../../../tables/settings/ErrorTable')) ); +const ImportSesssionTable = Loadable( + lazy(() => import('../../../../tables/settings/ImportSessionTable')) +); + const ProjectCodeTable = Loadable( lazy(() => import('../../../../tables/settings/ProjectCodeTable')) ); @@ -82,6 +87,12 @@ export default function AdminCenter() { icon: , content: }, + { + name: 'import', + label: t`Data Import`, + icon: , + content: + }, { name: 'background', label: t`Background Tasks`, diff --git a/src/frontend/src/tables/settings/ImportSessionTable.tsx b/src/frontend/src/tables/settings/ImportSessionTable.tsx new file mode 100644 index 000000000000..74e32f220885 --- /dev/null +++ b/src/frontend/src/tables/settings/ImportSessionTable.tsx @@ -0,0 +1,80 @@ +import { t } from '@lingui/macro'; +import { useCallback, useMemo } from 'react'; + +import { AttachmentLink } from '../../components/items/AttachmentLink'; +import { ProgressBar } from '../../components/items/ProgressBar'; +import { RenderUser } from '../../components/render/User'; +import { ApiEndpoints } from '../../enums/ApiEndpoints'; +import { ModelType } from '../../enums/ModelType'; +import { useTable } from '../../hooks/UseTable'; +import { apiUrl } from '../../states/ApiState'; +import { useUserState } from '../../states/UserState'; +import { TableColumn } from '../Column'; +import { DateColumn, StatusColumn } from '../ColumnRenderers'; +import { InvenTreeTable } from '../InvenTreeTable'; +import { RowAction } from '../RowActions'; + +export default function ImportSesssionTable() { + const table = useTable('importsession'); + const user = useUserState(); + + const columns: TableColumn[] = useMemo(() => { + return [ + { + accessor: 'model_type', + sortable: true + }, + StatusColumn({ model: ModelType.importsession }), + { + accessor: 'data_file', + render: (record: any) => ( + + ), + sortable: false + }, + DateColumn({ + accessor: 'timestamp', + title: t`Uploaded` + }), + { + accessor: 'user', + sortable: false, + render: (record: any) => RenderUser({ instance: record.user_detail }) + }, + { + sortable: false, + accessor: 'row_count', + title: t`Imported Rows`, + render: (record: any) => ( + + ) + } + ]; + }, []); + + const tableActions = useMemo(() => { + return []; + }, []); + + const rowActions = useCallback((record: any): RowAction[] => { + return []; + }, []); + + return ( + <> + + + ); +} diff --git a/src/frontend/src/tables/settings/ProjectCodeTable.tsx b/src/frontend/src/tables/settings/ProjectCodeTable.tsx index d68923f63fae..8a20d161f936 100644 --- a/src/frontend/src/tables/settings/ProjectCodeTable.tsx +++ b/src/frontend/src/tables/settings/ProjectCodeTable.tsx @@ -86,16 +86,12 @@ export default function ProjectCodeTable() { ); const tableActions = useMemo(() => { - let actions = []; - - actions.push( + return [ newProjectCode.open()} tooltip={t`Add project code`} /> - ); - - return actions; + ]; }, []); return ( From 089a288a1a55c4775803de2b45b2c8e16c01897b Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 09:50:27 +0000 Subject: [PATCH 127/190] Fix for table filter functions - New mantine version requires string values --- src/frontend/src/hooks/UseFilter.tsx | 10 +++++++++- src/frontend/src/tables/Filter.tsx | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/frontend/src/hooks/UseFilter.tsx b/src/frontend/src/hooks/UseFilter.tsx index 2ae5399c04f7..b10a151082f6 100644 --- a/src/frontend/src/hooks/UseFilter.tsx +++ b/src/frontend/src/hooks/UseFilter.tsx @@ -44,7 +44,15 @@ export function useFilters(props: UseFilterProps) { }); const choices: TableFilterChoice[] = useMemo(() => { - return query.data?.map(props.transform) ?? []; + let opts = query.data?.map(props.transform) ?? []; + + // Ensure stringiness + return opts.map((opt: any) => { + return { + value: opt.value.toString(), + label: opt?.label?.toString() ?? opt.value.toString() + }; + }); }, [props.transform, query.data]); const refresh = useCallback(() => { diff --git a/src/frontend/src/tables/Filter.tsx b/src/frontend/src/tables/Filter.tsx index cf6d7aabe987..948ef95634df 100644 --- a/src/frontend/src/tables/Filter.tsx +++ b/src/frontend/src/tables/Filter.tsx @@ -72,8 +72,8 @@ export function StatusFilterOptions( return Object.keys(codes).map((key) => { const entry = codes[key]; return { - value: entry.key, - label: entry.label ?? entry.key + value: entry.key.toString(), + label: entry.label?.toString() ?? entry.key.toString() }; }); } From 65dc3c7617432242d14ab7814df5a724fd7c4719 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 09:50:47 +0000 Subject: [PATCH 128/190] Add filters for import session table --- src/backend/InvenTree/importer/api.py | 2 +- .../tables/settings/ImportSessionTable.tsx | 42 ++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/backend/InvenTree/importer/api.py b/src/backend/InvenTree/importer/api.py index 7f6377a0c62b..25aa5e7a6c5d 100644 --- a/src/backend/InvenTree/importer/api.py +++ b/src/backend/InvenTree/importer/api.py @@ -50,7 +50,7 @@ class DataImportSessionList(ListCreateAPI): filter_backends = SEARCH_ORDER_FILTER - filterset_fields = ['model_type', 'status'] + filterset_fields = ['model_type', 'status', 'user'] ordering_fields = ['timestamp', 'status', 'model_type'] diff --git a/src/frontend/src/tables/settings/ImportSessionTable.tsx b/src/frontend/src/tables/settings/ImportSessionTable.tsx index 74e32f220885..626015608ae0 100644 --- a/src/frontend/src/tables/settings/ImportSessionTable.tsx +++ b/src/frontend/src/tables/settings/ImportSessionTable.tsx @@ -6,11 +6,13 @@ import { ProgressBar } from '../../components/items/ProgressBar'; import { RenderUser } from '../../components/render/User'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { ModelType } from '../../enums/ModelType'; +import { useFilters, useUserFilters } from '../../hooks/UseFilter'; import { useTable } from '../../hooks/UseTable'; import { apiUrl } from '../../states/ApiState'; import { useUserState } from '../../states/UserState'; import { TableColumn } from '../Column'; import { DateColumn, StatusColumn } from '../ColumnRenderers'; +import { StatusFilterOptions, TableFilter } from '../Filter'; import { InvenTreeTable } from '../InvenTreeTable'; import { RowAction } from '../RowActions'; @@ -56,6 +58,43 @@ export default function ImportSesssionTable() { ]; }, []); + const userFilter = useUserFilters(); + + const modelTypeFilters = useFilters({ + url: apiUrl(ApiEndpoints.import_session_list), + method: 'OPTIONS', + accessor: 'data.actions.POST.model_type.choices', + transform: (item: any) => { + return { + value: item.value, + label: item.display_name + }; + } + }); + + const tableFilters: TableFilter[] = useMemo(() => { + return [ + { + name: 'model_type', + label: t`Model Type`, + description: t`Filter by target model type`, + choices: modelTypeFilters.choices + }, + { + name: 'status', + label: t`Status`, + description: t`Filter by import session status`, + choiceFunction: StatusFilterOptions(ModelType.importsession) + }, + { + name: 'user', + label: t`User`, + description: t`Filter by user`, + choices: userFilter.choices + } + ]; + }, [modelTypeFilters.choices, userFilter.choices]); + const tableActions = useMemo(() => { return []; }, []); @@ -72,7 +111,8 @@ export default function ImportSesssionTable() { columns={columns} props={{ rowActions: rowActions, - tableActions: tableActions + tableActions: tableActions, + tableFilters: tableFilters }} /> From 95d9c5fc56601ac19105860ef9329a59c44229d2 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 11:36:48 +0000 Subject: [PATCH 129/190] Remove debug message --- src/backend/InvenTree/InvenTree/metadata.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/metadata.py b/src/backend/InvenTree/InvenTree/metadata.py index 3eaec8e17122..bde8def86d62 100644 --- a/src/backend/InvenTree/InvenTree/metadata.py +++ b/src/backend/InvenTree/InvenTree/metadata.py @@ -200,11 +200,6 @@ def get_serializer_info(self, serializer): field_value = serializer_info[name].get(field_key, None) model_value = getattr(field, model_key, None) - if name == 'category': - print('Category field:', field_key, model_key) - print(' - field_value:', field_value) - print(' - model_value:', model_value) - if value := self.override_value(name, field_value, model_value): serializer_info[name][field_key] = value From e204fe0d6eb23eec1f7e0876aaf0d1ccd2845517 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 11:40:37 +0000 Subject: [PATCH 130/190] fix for --- src/frontend/src/tables/FilterSelectDrawer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/tables/FilterSelectDrawer.tsx b/src/frontend/src/tables/FilterSelectDrawer.tsx index 2b79e30956b6..57b5d10a0057 100644 --- a/src/frontend/src/tables/FilterSelectDrawer.tsx +++ b/src/frontend/src/tables/FilterSelectDrawer.tsx @@ -41,7 +41,7 @@ function FilterItem({ return ( - + {flt.label} {flt.description} From 00826001bfe0a084cfe54f6f76602df5b94ddef8 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 11:50:10 +0000 Subject: [PATCH 131/190] Create new import session from admin page --- .../tables/settings/ImportSessionTable.tsx | 61 +++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/src/frontend/src/tables/settings/ImportSessionTable.tsx b/src/frontend/src/tables/settings/ImportSessionTable.tsx index 626015608ae0..ba898165ff8b 100644 --- a/src/frontend/src/tables/settings/ImportSessionTable.tsx +++ b/src/frontend/src/tables/settings/ImportSessionTable.tsx @@ -1,12 +1,19 @@ import { t } from '@lingui/macro'; -import { useCallback, useMemo } from 'react'; +import { useCallback, useMemo, useState } from 'react'; +import { AddItemButton } from '../../components/buttons/AddItemButton'; +import ImporterDrawer from '../../components/importer/ImporterDrawer'; import { AttachmentLink } from '../../components/items/AttachmentLink'; import { ProgressBar } from '../../components/items/ProgressBar'; import { RenderUser } from '../../components/render/User'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { ModelType } from '../../enums/ModelType'; +import { dataImporterSessionFields } from '../../forms/ImporterForms'; import { useFilters, useUserFilters } from '../../hooks/UseFilter'; +import { + useCreateApiFormModal, + useDeleteApiFormModal +} from '../../hooks/UseForm'; import { useTable } from '../../hooks/UseTable'; import { apiUrl } from '../../states/ApiState'; import { useUserState } from '../../states/UserState'; @@ -14,12 +21,36 @@ import { TableColumn } from '../Column'; import { DateColumn, StatusColumn } from '../ColumnRenderers'; import { StatusFilterOptions, TableFilter } from '../Filter'; import { InvenTreeTable } from '../InvenTreeTable'; -import { RowAction } from '../RowActions'; +import { RowAction, RowDeleteAction } from '../RowActions'; export default function ImportSesssionTable() { const table = useTable('importsession'); const user = useUserState(); + const [opened, setOpened] = useState(false); + + const [selectedSession, setSelectedSession] = useState( + undefined + ); + + const deleteSession = useDeleteApiFormModal({ + url: ApiEndpoints.import_session_list, + pk: selectedSession, + title: t`Delete Import Session`, + table: table + }); + + const newImportSession = useCreateApiFormModal({ + url: ApiEndpoints.import_session_list, + title: t`Create Import Session`, + fields: dataImporterSessionFields(), + onFormSuccess: (response: any) => { + setSelectedSession(response.pk); + setOpened(true); + table.refreshTable(); + } + }); + const columns: TableColumn[] = useMemo(() => { return [ { @@ -96,15 +127,29 @@ export default function ImportSesssionTable() { }, [modelTypeFilters.choices, userFilter.choices]); const tableActions = useMemo(() => { - return []; + return [ + newImportSession.open()} + /> + ]; }, []); const rowActions = useCallback((record: any): RowAction[] => { - return []; + return [ + RowDeleteAction({ + onClick: () => { + setSelectedSession(record.pk); + deleteSession.open(); + } + }) + ]; }, []); return ( <> + {newImportSession.modal} + {deleteSession.modal} + { + setSelectedSession(undefined); + setOpened(false); + }} + /> ); } From 41b7fcd905972b561e0e0fdb574587d370e6d1e6 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 11:50:45 +0000 Subject: [PATCH 132/190] Cleanup playground --- src/frontend/src/pages/Index/Playground.tsx | 47 +-------------------- 1 file changed, 1 insertion(+), 46 deletions(-) diff --git a/src/frontend/src/pages/Index/Playground.tsx b/src/frontend/src/pages/Index/Playground.tsx index 5326aeced637..8ce44f54f8b1 100644 --- a/src/frontend/src/pages/Index/Playground.tsx +++ b/src/frontend/src/pages/Index/Playground.tsx @@ -10,17 +10,15 @@ import { } from '@mantine/core'; import { SpotlightActionData } from '@mantine/spotlight'; import { IconAlien } from '@tabler/icons-react'; -import { ReactNode, useCallback, useMemo, useState } from 'react'; +import { ReactNode, useMemo, useState } from 'react'; import { OptionsApiForm } from '../../components/forms/ApiForm'; -import ImporterDrawer from '../../components/importer/ImporterDrawer'; import { PlaceholderPill } from '../../components/items/Placeholder'; import { StylishText } from '../../components/items/StylishText'; import { firstSpotlight } from '../../components/nav/Layout'; import { StatusRenderer } from '../../components/render/StatusRenderer'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { ModelType } from '../../enums/ModelType'; -import { dataImporterSessionFields } from '../../forms/ImporterForms'; import { partCategoryFields, usePartFields } from '../../forms/PartForms'; import { useCreateStockItem } from '../../forms/StockForms'; import { @@ -161,45 +159,6 @@ function StatusLabelPlayground() { ); } -// Data importing -function DataImportingPlayground() { - const [opened, setOpened] = useState(false); - - const importSessionFields = dataImporterSessionFields(); - - const [importSessionId, setImportSessionId] = useState(0); - - const createNewImportSession = useCreateApiFormModal({ - url: ApiEndpoints.import_session_list, - title: 'Create Import Session', - fields: importSessionFields, - initialData: { - parent: 1, - model: 'partcategory' - }, - onFormSuccess: (response: any) => { - setImportSessionId(response.pk); - setOpened(true); - } - }); - - const openDrawer = useCallback(() => { - createNewImportSession.open(); - }, []); - - return ( - <> - {createNewImportSession.modal} - - setOpened(false)} - /> - - ); -} - // Sample for spotlight actions function SpotlighPlayground() { return ( @@ -274,10 +233,6 @@ export default function Playground() { title="Status labels" content={} /> - } - /> } From 501e93ce6d3fff04763114f0755d63226a89c9ac Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 26 Jun 2024 11:52:48 +0000 Subject: [PATCH 133/190] Re-open an existing import session --- .../src/components/importer/ImporterDrawer.tsx | 10 +--------- src/frontend/src/hooks/UseImportSession.tsx | 7 ------- .../src/tables/settings/ImportSessionTable.tsx | 6 +++++- 3 files changed, 6 insertions(+), 17 deletions(-) diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index 281ee335d266..cee5c2c08667 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -93,14 +93,6 @@ export default function ImporterDrawer({ } }, [session.status]); - const cancelImport = useCallback(() => { - // Cancel import session by deleting on the server - session.cancelSession(); - - // Close the modal - onClose(); - }, [session]); - const title: ReactNode = useMemo(() => { return ( @@ -116,7 +108,7 @@ export default function ImporterDrawer({ - + diff --git a/src/frontend/src/hooks/UseImportSession.tsx b/src/frontend/src/hooks/UseImportSession.tsx index d28979079439..7da22ea65514 100644 --- a/src/frontend/src/hooks/UseImportSession.tsx +++ b/src/frontend/src/hooks/UseImportSession.tsx @@ -21,7 +21,6 @@ export enum ImportSessionStatus { export type ImportSessionState = { sessionId: number; sessionData: any; - cancelSession: () => void; refreshSession: () => void; sessionQuery: any; status: ImportSessionStatus; @@ -47,11 +46,6 @@ export function useImportSession({ defaultValue: {} }); - // Cancel the importer session (by deleting it) - const cancelSession = useCallback(() => { - api.delete(apiUrl(ApiEndpoints.import_session_list, sessionId)); - }, [sessionId]); - // Current step of the import process const status: ImportSessionStatus = useMemo(() => { return sessionData?.status ?? ImportSessionStatus.INITIAL; @@ -102,7 +96,6 @@ export function useImportSession({ return { sessionData, sessionId, - cancelSession, refreshSession, sessionQuery, status, diff --git a/src/frontend/src/tables/settings/ImportSessionTable.tsx b/src/frontend/src/tables/settings/ImportSessionTable.tsx index ba898165ff8b..5326acb735c3 100644 --- a/src/frontend/src/tables/settings/ImportSessionTable.tsx +++ b/src/frontend/src/tables/settings/ImportSessionTable.tsx @@ -157,7 +157,11 @@ export default function ImportSesssionTable() { props={{ rowActions: rowActions, tableActions: tableActions, - tableFilters: tableFilters + tableFilters: tableFilters, + onRowClick: (record: any) => { + setSelectedSession(record.pk); + setOpened(true); + } }} /> Date: Wed, 26 Jun 2024 11:59:47 +0000 Subject: [PATCH 134/190] Memoize cell value --- src/frontend/src/components/importer/ImportDataSelector.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 5de11ca4fb75..369043e9b58f 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -46,11 +46,15 @@ function ImporterDataCell({ return row?.errors[column.field] ?? []; }, [row.errors, column.field]); + const cellValue = useMemo(() => { + return row.data ? row.data[column.field] : ''; + }, [row.data, column.field]); + return ( - {row.data[column.field]} + {cellValue} {cellErrors.map((error: string) => ( {error} From 743e91ef3e0ac4af8457b3a0498b87f5b4870ef8 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 28 Jun 2024 23:35:38 +0000 Subject: [PATCH 135/190] Update --- .../importer/ImportDataSelector.tsx | 44 +++++++++++++------ 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 369043e9b58f..e06dc363f50c 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -1,5 +1,12 @@ import { t } from '@lingui/macro'; -import { ActionIcon, Group, Stack, Text, Tooltip } from '@mantine/core'; +import { + ActionIcon, + Group, + HoverCard, + Stack, + Text, + Tooltip +} from '@mantine/core'; import { IconCircleCheck, IconEdit, @@ -50,26 +57,37 @@ function ImporterDataCell({ return row.data ? row.data[column.field] : ''; }, [row.data, column.field]); + const cellValid: boolean = useMemo( + () => cellErrors.length == 0, + [cellErrors] + ); + return ( - - + + + + + {cellValue} + +
+ + + + + +
+ + + - {cellValue} {cellErrors.map((error: string) => ( {error} ))} - -
- - - - - -
- + + ); } From f90d802f8681d6def3666d9fd3d7238272603f35 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sat, 29 Jun 2024 10:42:37 +0000 Subject: [PATCH 136/190] Enable download of build line data --- src/backend/InvenTree/build/serializers.py | 21 +++++++++++++++---- .../templates/js/translated/build.js | 1 + 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/backend/InvenTree/build/serializers.py b/src/backend/InvenTree/build/serializers.py index db858877b480..2262f4c7780d 100644 --- a/src/backend/InvenTree/build/serializers.py +++ b/src/backend/InvenTree/build/serializers.py @@ -1132,6 +1132,10 @@ def __init__(self, *args, **kwargs): class BuildLineSerializer(DataImportExportSerializerMixin, InvenTreeModelSerializer): """Serializer for a BuildItem object.""" + export_exclude_fields = [ + 'allocations', + ] + class Meta: """Serializer metaclass""" @@ -1145,6 +1149,11 @@ class Meta: 'quantity', 'allocations', + # Part detail fields + 'part', + 'part_name', + 'part_IPN', + # Annotated fields 'allocated', 'in_production', @@ -1162,6 +1171,10 @@ class Meta: 'allocations', ] + part = serializers.PrimaryKeyRelatedField(source='bom_item.sub_part', many=False, read_only=True) + part_name = serializers.CharField(source='bom_item.sub_part.full_name', read_only=True) + part_IPN = serializers.CharField(source='bom_item.sub_part.IPN', read_only=True) + quantity = serializers.FloatField() bom_item = serializers.PrimaryKeyRelatedField(label=_('BOM Item'), read_only=True) @@ -1192,10 +1205,10 @@ class Meta: read_only=True ) - available_substitute_stock = serializers.FloatField(read_only=True) - available_variant_stock = serializers.FloatField(read_only=True) - total_available_stock = serializers.FloatField(read_only=True) - external_stock = serializers.FloatField(read_only=True) + available_substitute_stock = serializers.FloatField(read_only=True, label=_('Available Substitute Stock')) + available_variant_stock = serializers.FloatField(read_only=True, label=_('Available Variant Stock')) + total_available_stock = serializers.FloatField(read_only=True, label=_('Total Available Stock')) + external_stock = serializers.FloatField(read_only=True, label=_('External Stock')) @staticmethod def annotate_queryset(queryset, build=None): diff --git a/src/backend/InvenTree/templates/js/translated/build.js b/src/backend/InvenTree/templates/js/translated/build.js index f761296f8f38..360b430cf347 100644 --- a/src/backend/InvenTree/templates/js/translated/build.js +++ b/src/backend/InvenTree/templates/js/translated/build.js @@ -2449,6 +2449,7 @@ function loadBuildLineTable(table, build_id, options={}) { // If data is passed directly to this function, do not setup filters if (!options.data) { setupFilterList('buildlines', $(table), filterTarget, { + download: true, labels: { modeltype: 'buildline', }, From 0588de4187191495601ae96600c3e6b331b66fce Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sat, 29 Jun 2024 10:47:25 +0000 Subject: [PATCH 137/190] Add extra detail fields --- src/backend/InvenTree/build/serializers.py | 21 +++++++++++++++---- .../src/tables/build/BuildLineTable.tsx | 3 ++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/backend/InvenTree/build/serializers.py b/src/backend/InvenTree/build/serializers.py index 2262f4c7780d..8e2bf2ccff58 100644 --- a/src/backend/InvenTree/build/serializers.py +++ b/src/backend/InvenTree/build/serializers.py @@ -1149,6 +1149,12 @@ class Meta: 'quantity', 'allocations', + # BOM item detail fields + 'reference', + 'consumable', + 'optional', + 'trackable', + # Part detail fields 'part', 'part_name', @@ -1171,11 +1177,18 @@ class Meta: 'allocations', ] - part = serializers.PrimaryKeyRelatedField(source='bom_item.sub_part', many=False, read_only=True) - part_name = serializers.CharField(source='bom_item.sub_part.full_name', read_only=True) - part_IPN = serializers.CharField(source='bom_item.sub_part.IPN', read_only=True) + # Part info fields + part = serializers.PrimaryKeyRelatedField(source='bom_item.sub_part', label=_('Part'), many=False, read_only=True) + part_name = serializers.CharField(source='bom_item.sub_part.name', label=_('Part Name'), read_only=True) + part_IPN = serializers.CharField(source='bom_item.sub_part.IPN', label=_('Part IPN'), read_only=True) + + # BOM item info fields + reference = serializers.CharField(source='bom_item.reference', label=_('Reference'), read_only=True) + consumable = serializers.BooleanField(source='bom_item.consumable', label=_('Consumable'), read_only=True) + optional = serializers.BooleanField(source='bom_item.optional', label=_('Optional'), read_only=True) + trackable = serializers.BooleanField(source='bom_item.sub_part.trackable', label=_('Trackable'), read_only=True) - quantity = serializers.FloatField() + quantity = serializers.FloatField(label=_('Quantity')) bom_item = serializers.PrimaryKeyRelatedField(label=_('BOM Item'), read_only=True) diff --git a/src/frontend/src/tables/build/BuildLineTable.tsx b/src/frontend/src/tables/build/BuildLineTable.tsx index 3e337aae703d..6daef80c99c5 100644 --- a/src/frontend/src/tables/build/BuildLineTable.tsx +++ b/src/frontend/src/tables/build/BuildLineTable.tsx @@ -256,7 +256,8 @@ export default function BuildLineTable({ params = {} }: { params?: any }) { tableFilters: tableFilters, rowActions: rowActions, modelType: ModelType.part, - modelField: 'part_detail.pk' + modelField: 'part_detail.pk', + enableDownload: true }} /> ); From ddccc0d5857d6cb436612ee1449d3725e50965e6 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 02:17:52 +0000 Subject: [PATCH 138/190] Register data importers for the stock app --- src/backend/InvenTree/stock/serializers.py | 13 +++++++++++-- .../src/components/importer/ImportDataSelector.tsx | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/backend/InvenTree/stock/serializers.py b/src/backend/InvenTree/stock/serializers.py index f039b506c31e..fd3390e828ee 100644 --- a/src/backend/InvenTree/stock/serializers.py +++ b/src/backend/InvenTree/stock/serializers.py @@ -27,6 +27,7 @@ from common.settings import get_global_setting from company.serializers import SupplierPartSerializer from importer.mixins import DataImportExportSerializerMixin +from importer.registry import register_importer from InvenTree.serializers import InvenTreeCurrencySerializer, InvenTreeDecimalField from part.serializers import PartBriefSerializer, PartTestTemplateSerializer @@ -178,7 +179,10 @@ class Meta: fields = ['pk', 'name', 'pathstring'] -class StockItemTestResultSerializer(InvenTree.serializers.InvenTreeModelSerializer): +@register_importer() +class StockItemTestResultSerializer( + DataImportExportSerializerMixin, InvenTree.serializers.InvenTreeModelSerializer +): """Serializer for the StockItemTestResult model.""" class Meta: @@ -317,6 +321,7 @@ def validate_serial(self, value): return value +@register_importer() class StockItemSerializer( DataImportExportSerializerMixin, InvenTree.serializers.InvenTreeTagModelSerializer ): @@ -1034,6 +1039,7 @@ def annotate_queryset(queryset): return queryset.annotate(sublocations=stock.filters.annotate_sub_locations()) +@register_importer() class LocationSerializer( DataImportExportSerializerMixin, InvenTree.serializers.InvenTreeTagModelSerializer ): @@ -1119,7 +1125,10 @@ def annotate_queryset(queryset): ) -class StockTrackingSerializer(InvenTree.serializers.InvenTreeModelSerializer): +@register_importer() +class StockTrackingSerializer( + DataImportExportSerializerMixin, InvenTree.serializers.InvenTreeModelSerializer +): """Serializer for StockItemTracking model.""" class Meta: diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index e06dc363f50c..8171f34c2f6f 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -81,7 +81,7 @@ function ImporterDataCell({ {cellErrors.map((error: string) => ( - + {error} ))} From f48b9310b55eae4a778392df0ffb791248962ad8 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 02:20:20 +0000 Subject: [PATCH 139/190] Enable download of stock item tracking data --- src/backend/InvenTree/part/api.py | 2 +- src/backend/InvenTree/stock/api.py | 3 +-- src/frontend/src/tables/stock/StockTrackingTable.tsx | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/InvenTree/part/api.py b/src/backend/InvenTree/part/api.py index 0bddcc424178..6d088f661f33 100644 --- a/src/backend/InvenTree/part/api.py +++ b/src/backend/InvenTree/part/api.py @@ -29,7 +29,7 @@ InvenTreeDateFilter, InvenTreeSearchFilter, ) -from InvenTree.helpers import DownloadFile, increment_serial_number, isNull, str2bool +from InvenTree.helpers import increment_serial_number, isNull, str2bool from InvenTree.mixins import ( CreateAPI, CustomRetrieveUpdateDestroyAPI, diff --git a/src/backend/InvenTree/stock/api.py b/src/backend/InvenTree/stock/api.py index 482eb04d67dc..3e93d8eb3380 100644 --- a/src/backend/InvenTree/stock/api.py +++ b/src/backend/InvenTree/stock/api.py @@ -37,7 +37,6 @@ InvenTreeDateFilter, ) from InvenTree.helpers import ( - DownloadFile, extract_serial_numbers, generateTestKey, is_ajax, @@ -1350,7 +1349,7 @@ class StockTrackingDetail(RetrieveAPI): serializer_class = StockSerializers.StockTrackingSerializer -class StockTrackingList(ListAPI): +class StockTrackingList(DataExportViewMixin, ListAPI): """API endpoint for list view of StockItemTracking objects. StockItemTracking objects are read-only diff --git a/src/frontend/src/tables/stock/StockTrackingTable.tsx b/src/frontend/src/tables/stock/StockTrackingTable.tsx index 208c6f2b984a..2a15d592b71d 100644 --- a/src/frontend/src/tables/stock/StockTrackingTable.tsx +++ b/src/frontend/src/tables/stock/StockTrackingTable.tsx @@ -213,7 +213,8 @@ export function StockTrackingTable({ itemId }: { itemId: number }) { params: { item: itemId, user_detail: true - } + }, + enableDownload: true }} /> ); From 0055c9ac3126e750279a5d252fb12b172b5b3803 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 02:32:17 +0000 Subject: [PATCH 140/190] Register importerrs for "company" app --- .../InvenTree/company/migrations/0001_initial.py | 4 ++++ .../migrations/0038_manufacturerpartparameter.py | 1 + .../company/migrations/0066_auto_20230616_2059.py | 5 ++++- src/backend/InvenTree/company/models.py | 10 +++++++++- src/backend/InvenTree/company/serializers.py | 12 +++++++++++- .../InvenTree/stock/migrations/0001_initial.py | 3 +++ .../stock/migrations/0040_stockitemtestresult.py | 3 +++ src/backend/InvenTree/stock/models.py | 10 ++++++++++ 8 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/backend/InvenTree/company/migrations/0001_initial.py b/src/backend/InvenTree/company/migrations/0001_initial.py index cfc73bea2066..6c6f31f9386e 100644 --- a/src/backend/InvenTree/company/migrations/0001_initial.py +++ b/src/backend/InvenTree/company/migrations/0001_initial.py @@ -44,6 +44,9 @@ class Migration(migrations.Migration): ('email', models.EmailField(blank=True, max_length=254)), ('role', models.CharField(blank=True, max_length=100)), ], + options={ + 'verbose_name': 'Contact', + } ), migrations.CreateModel( name='SupplierPart', @@ -75,6 +78,7 @@ class Migration(migrations.Migration): ('part', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pricebreaks', to='company.SupplierPart')), ], options={ + 'verbose_name': 'Supplier Price Break', 'db_table': 'part_supplierpricebreak', }, ), diff --git a/src/backend/InvenTree/company/migrations/0038_manufacturerpartparameter.py b/src/backend/InvenTree/company/migrations/0038_manufacturerpartparameter.py index dccfa715e891..dd833ccfa3fe 100644 --- a/src/backend/InvenTree/company/migrations/0038_manufacturerpartparameter.py +++ b/src/backend/InvenTree/company/migrations/0038_manufacturerpartparameter.py @@ -21,6 +21,7 @@ class Migration(migrations.Migration): ('manufacturer_part', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='parameters', to='company.manufacturerpart', verbose_name='Manufacturer Part')), ], options={ + 'verbose_name': 'Manufacturer Part Parameter', 'unique_together': {('manufacturer_part', 'name')}, }, ), diff --git a/src/backend/InvenTree/company/migrations/0066_auto_20230616_2059.py b/src/backend/InvenTree/company/migrations/0066_auto_20230616_2059.py index 19ce7983013a..f5160f3255a5 100644 --- a/src/backend/InvenTree/company/migrations/0066_auto_20230616_2059.py +++ b/src/backend/InvenTree/company/migrations/0066_auto_20230616_2059.py @@ -12,7 +12,10 @@ class Migration(migrations.Migration): operations = [ migrations.AlterModelOptions( name='address', - options={'verbose_name_plural': 'Addresses'}, + options={ + 'verbose_name': 'Address', + 'verbose_name_plural': 'Addresses' + }, ), migrations.AlterField( model_name='address', diff --git a/src/backend/InvenTree/company/models.py b/src/backend/InvenTree/company/models.py index 8683be248e16..3f5b5298b3b0 100644 --- a/src/backend/InvenTree/company/models.py +++ b/src/backend/InvenTree/company/models.py @@ -269,6 +269,11 @@ class Contact(InvenTree.models.InvenTreeMetadataModel): role: position in company """ + class Meta: + """Metaclass defines extra model options.""" + + verbose_name = _('Contact') + @staticmethod def get_api_url(): """Return the API URL associated with the Contcat model.""" @@ -306,7 +311,8 @@ class Address(InvenTree.models.InvenTreeModel): class Meta: """Metaclass defines extra model options.""" - verbose_name_plural = 'Addresses' + verbose_name = _('Address') + verbose_name_plural = _('Addresses') def __init__(self, *args, **kwargs): """Custom init function.""" @@ -560,6 +566,7 @@ class ManufacturerPartParameter(InvenTree.models.InvenTreeModel): class Meta: """Metaclass defines extra model options.""" + verbose_name = _('Manufacturer Part Parameter') unique_together = ('manufacturer_part', 'name') @staticmethod @@ -1005,6 +1012,7 @@ class SupplierPriceBreak(common.models.PriceBreak): class Meta: """Metaclass defines extra model options.""" + verbose_name = _('Supplier Price Break') unique_together = ('part', 'quantity') # This model was moved from the 'Part' app diff --git a/src/backend/InvenTree/company/serializers.py b/src/backend/InvenTree/company/serializers.py index da843a8c3d58..a0fba43fa640 100644 --- a/src/backend/InvenTree/company/serializers.py +++ b/src/backend/InvenTree/company/serializers.py @@ -11,6 +11,7 @@ import part.filters from importer.mixins import DataImportExportSerializerMixin +from importer.registry import register_importer from InvenTree.serializers import ( InvenTreeCurrencySerializer, InvenTreeDecimalField, @@ -57,6 +58,7 @@ class Meta: thumbnail = serializers.CharField(source='get_thumbnail_url', read_only=True) +@register_importer() class AddressSerializer(DataImportExportSerializerMixin, InvenTreeModelSerializer): """Serializer for the Address Model.""" @@ -101,6 +103,7 @@ class Meta: ] +@register_importer() class CompanySerializer( DataImportExportSerializerMixin, NotesFieldMixin, @@ -191,6 +194,7 @@ def save(self): return self.instance +@register_importer() class ContactSerializer(DataImportExportSerializerMixin, InvenTreeModelSerializer): """Serializer class for the Contact model.""" @@ -205,6 +209,7 @@ class Meta: ) +@register_importer() class ManufacturerPartSerializer( DataImportExportSerializerMixin, InvenTreeTagModelSerializer ): @@ -260,7 +265,10 @@ def __init__(self, *args, **kwargs): ) -class ManufacturerPartParameterSerializer(InvenTreeModelSerializer): +@register_importer() +class ManufacturerPartParameterSerializer( + DataImportExportSerializerMixin, InvenTreeModelSerializer +): """Serializer for the ManufacturerPartParameter model.""" class Meta: @@ -291,6 +299,7 @@ def __init__(self, *args, **kwargs): ) +@register_importer() class SupplierPartSerializer( DataImportExportSerializerMixin, InvenTreeTagModelSerializer ): @@ -451,6 +460,7 @@ def create(self, validated_data): return supplier_part +@register_importer() class SupplierPriceBreakSerializer( DataImportExportSerializerMixin, InvenTreeModelSerializer ): diff --git a/src/backend/InvenTree/stock/migrations/0001_initial.py b/src/backend/InvenTree/stock/migrations/0001_initial.py index 040a48efeb3f..ac8b7b7a488a 100644 --- a/src/backend/InvenTree/stock/migrations/0001_initial.py +++ b/src/backend/InvenTree/stock/migrations/0001_initial.py @@ -64,6 +64,9 @@ class Migration(migrations.Migration): ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tracking_info', to='stock.StockItem')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), ], + options={ + 'verbose_name': 'Stock Item Tracking', + } ), migrations.AddField( model_name='stockitem', diff --git a/src/backend/InvenTree/stock/migrations/0040_stockitemtestresult.py b/src/backend/InvenTree/stock/migrations/0040_stockitemtestresult.py index fdf0344925fe..6629d6634ddc 100644 --- a/src/backend/InvenTree/stock/migrations/0040_stockitemtestresult.py +++ b/src/backend/InvenTree/stock/migrations/0040_stockitemtestresult.py @@ -25,5 +25,8 @@ class Migration(migrations.Migration): ('stock_item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='test_results', to='stock.StockItem')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), ], + options={ + 'verbose_name': 'Stock Item Test Result', + }, ), ] diff --git a/src/backend/InvenTree/stock/models.py b/src/backend/InvenTree/stock/models.py index f587cdf71f6e..0b7b312fb9b0 100644 --- a/src/backend/InvenTree/stock/models.py +++ b/src/backend/InvenTree/stock/models.py @@ -2276,6 +2276,11 @@ class StockItemTracking(InvenTree.models.InvenTreeModel): deltas: The changes associated with this history item """ + class Meta: + """Meta data for the StockItemTracking class.""" + + verbose_name = _('Stock Item Tracking') + @staticmethod def get_api_url(): """Return API url.""" @@ -2344,6 +2349,11 @@ class StockItemTestResult(InvenTree.models.InvenTreeMetadataModel): date: Date the test result was recorded """ + class Meta: + """Meta data for the StockItemTestResult class.""" + + verbose_name = _('Stock Item Test Result') + def __str__(self): """Return string representation.""" return f'{self.test_name} - {self.result}' From 843dbda12bd00942ab34bec8cc883aec238bd252 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 02:36:25 +0000 Subject: [PATCH 141/190] Register importers for the "order" app --- src/backend/InvenTree/order/serializers.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/backend/InvenTree/order/serializers.py b/src/backend/InvenTree/order/serializers.py index ad2c63221305..e0fe096b69e4 100644 --- a/src/backend/InvenTree/order/serializers.py +++ b/src/backend/InvenTree/order/serializers.py @@ -34,6 +34,7 @@ SupplierPartSerializer, ) from importer.mixins import DataImportExportSerializerMixin +from importer.registry import register_importer from InvenTree.helpers import ( current_date, extract_serial_numbers, @@ -199,6 +200,7 @@ class AbstractExtraLineMeta: ] +@register_importer() class PurchaseOrderSerializer( NotesFieldMixin, TotalPriceMixin, AbstractOrderSerializer, InvenTreeModelSerializer ): @@ -341,6 +343,7 @@ def save(self): order.place_order() +@register_importer() class PurchaseOrderLineItemSerializer( DataImportExportSerializerMixin, InvenTreeModelSerializer ): @@ -518,6 +521,7 @@ def validate(self, data): return data +@register_importer() class PurchaseOrderExtraLineSerializer( AbstractExtraLineSerializer, InvenTreeModelSerializer ): @@ -760,6 +764,7 @@ def save(self): raise ValidationError(detail=serializers.as_serializer_error(exc)) +@register_importer() class SalesOrderSerializer( NotesFieldMixin, TotalPriceMixin, AbstractOrderSerializer, InvenTreeModelSerializer ): @@ -919,6 +924,7 @@ def __init__(self, *args, **kwargs): ) +@register_importer() class SalesOrderLineItemSerializer( DataImportExportSerializerMixin, InvenTreeModelSerializer ): @@ -1070,6 +1076,7 @@ def annotate_queryset(queryset): ) +@register_importer() class SalesOrderShipmentSerializer(NotesFieldMixin, InvenTreeModelSerializer): """Serializer for the SalesOrderShipment class.""" @@ -1506,6 +1513,7 @@ def save(self): allocation.save() +@register_importer() class SalesOrderExtraLineSerializer( AbstractExtraLineSerializer, InvenTreeModelSerializer ): @@ -1519,6 +1527,7 @@ class Meta(AbstractExtraLineMeta): order_detail = SalesOrderSerializer(source='order', many=False, read_only=True) +@register_importer() class ReturnOrderSerializer( NotesFieldMixin, AbstractOrderSerializer, TotalPriceMixin, InvenTreeModelSerializer ): @@ -1697,6 +1706,7 @@ def save(self): order.receive_line_item(line_item, location, request.user) +@register_importer() class ReturnOrderLineItemSerializer( DataImportExportSerializerMixin, InvenTreeModelSerializer ): @@ -1752,6 +1762,7 @@ def __init__(self, *args, **kwargs): price_currency = InvenTreeCurrencySerializer(help_text=_('Line price currency')) +@register_importer() class ReturnOrderExtraLineSerializer( AbstractExtraLineSerializer, InvenTreeModelSerializer ): From 5163ae45a4ea18ab7bdae406ae8259914598f7ac Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 02:46:32 +0000 Subject: [PATCH 142/190] Add extra fields to purchase order line item serializer --- src/backend/InvenTree/order/serializers.py | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/backend/InvenTree/order/serializers.py b/src/backend/InvenTree/order/serializers.py index e0fe096b69e4..4c6a9bdd45e5 100644 --- a/src/backend/InvenTree/order/serializers.py +++ b/src/backend/InvenTree/order/serializers.py @@ -77,6 +77,8 @@ class TotalPriceMixin(serializers.Serializer): class AbstractOrderSerializer(DataImportExportSerializerMixin, serializers.Serializer): """Abstract serializer class which provides fields common to all order types.""" + export_exclude_fields = ['notes'] + # Number of line items in this order line_items = serializers.IntegerField(read_only=True, label=_('Line Items')) @@ -102,6 +104,10 @@ class AbstractOrderSerializer(DataImportExportSerializerMixin, serializers.Seria source='responsible', read_only=True, many=False ) + project_code = serializers.CharField( + source='project_code.code', label=_('Project Code'), read_only=True + ) + # Detail for project code field project_code_detail = ProjectCodeSerializer( source='project_code', read_only=True, many=False @@ -375,6 +381,11 @@ class Meta: 'total_price', 'link', 'merge_items', + 'sku', + 'mpn', + 'ipn', + 'internal_part', + 'internal_part_name', ] def __init__(self, *args, **kwargs): @@ -488,6 +499,25 @@ def validate_purchase_order(self, purchase_order): 'Merge items with the same part, destination and target date into one line item' ), default=True, + write_only=True, + ) + + sku = serializers.CharField(source='part.SKU', read_only=True, label=_('SKU')) + + mpn = serializers.CharField( + source='part.manufacturer_part.MPN', read_only=True, label=_('MPN') + ) + + ipn = serializers.CharField( + source='part.part.IPN', read_only=True, label=_('Internal Part Number') + ) + + internal_part = serializers.PrimaryKeyRelatedField( + source='part.part', read_only=True, many=False, label=_('Internal Part') + ) + + internal_part_name = serializers.CharField( + source='part.part.name', read_only=True, label=_('Internal Part Name') ) def validate(self, data): From 57382fe4923c7cf0474693bbe6dd90b50b97a9cf Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 03:48:30 +0000 Subject: [PATCH 143/190] Update verbose names for order models --- src/backend/InvenTree/InvenTree/settings.py | 2 +- src/backend/InvenTree/importer/registry.py | 4 +-- src/backend/InvenTree/importer/serializers.py | 6 ++++ .../order/migrations/0001_initial.py | 1 + .../migrations/0020_auto_20200420_0940.py | 1 + .../migrations/0024_salesorderallocation.py | 3 ++ .../migrations/0053_salesordershipment.py | 3 ++ ...chaseorderextraline_salesorderextraline.py | 2 ++ .../migrations/0083_returnorderextraline.py | 1 + .../migrations/0085_auto_20230322_1056.py | 1 + src/backend/InvenTree/order/models.py | 32 +++++++++++++++++++ 11 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/settings.py b/src/backend/InvenTree/InvenTree/settings.py index afb8298bce8f..cedeb4e215df 100644 --- a/src/backend/InvenTree/InvenTree/settings.py +++ b/src/backend/InvenTree/InvenTree/settings.py @@ -192,13 +192,13 @@ 'common.apps.CommonConfig', 'company.apps.CompanyConfig', 'plugin.apps.PluginAppConfig', # Plugin app runs before all apps that depend on the isPluginRegistryLoaded function - 'importer.apps.ImporterConfig', 'order.apps.OrderConfig', 'part.apps.PartConfig', 'report.apps.ReportConfig', 'stock.apps.StockConfig', 'users.apps.UsersConfig', 'machine.apps.MachineConfig', + 'importer.apps.ImporterConfig', 'web', 'generic', 'InvenTree.apps.InvenTreeConfig', # InvenTree app runs last diff --git a/src/backend/InvenTree/importer/registry.py b/src/backend/InvenTree/importer/registry.py index 9405af5bab66..2614c29ea5de 100644 --- a/src/backend/InvenTree/importer/registry.py +++ b/src/backend/InvenTree/importer/registry.py @@ -20,11 +20,11 @@ class DataImportSerializerRegister: def register(self, serializer) -> None: """Register a new serializer with the importer registry.""" if not issubclass(serializer, DataImportSerializerMixin): - logger.error('Invalid serializer class: %s', type(serializer)) + logger.debug('Invalid serializer class: %s', type(serializer)) return if not issubclass(serializer, Serializer): - logger.error('Invalid serializer class: %s', type(serializer)) + logger.debug('Invalid serializer class: %s', type(serializer)) return logger.debug('Registering serializer class for import: %s', type(serializer)) diff --git a/src/backend/InvenTree/importer/serializers.py b/src/backend/InvenTree/importer/serializers.py index 45449f54352a..aab1d28859d7 100644 --- a/src/backend/InvenTree/importer/serializers.py +++ b/src/backend/InvenTree/importer/serializers.py @@ -51,6 +51,12 @@ class Meta: ] read_only_fields = ['pk', 'user', 'status', 'columns'] + def __init__(self, *args, **kwargs): + """Override the constructor for the DataImportSession serializer.""" + super().__init__(*args, **kwargs) + + self.fields['model_type'].choices = importer.registry.supported_model_options() + data_file = InvenTreeAttachmentSerializerField() model_type = serializers.ChoiceField( diff --git a/src/backend/InvenTree/order/migrations/0001_initial.py b/src/backend/InvenTree/order/migrations/0001_initial.py index edceeffc1191..498aad298332 100644 --- a/src/backend/InvenTree/order/migrations/0001_initial.py +++ b/src/backend/InvenTree/order/migrations/0001_initial.py @@ -44,6 +44,7 @@ class Migration(migrations.Migration): ], options={ 'abstract': False, + 'verbose_name': 'Purchase Order Line Item' }, ), ] diff --git a/src/backend/InvenTree/order/migrations/0020_auto_20200420_0940.py b/src/backend/InvenTree/order/migrations/0020_auto_20200420_0940.py index 44d14014382c..083c300df3cb 100644 --- a/src/backend/InvenTree/order/migrations/0020_auto_20200420_0940.py +++ b/src/backend/InvenTree/order/migrations/0020_auto_20200420_0940.py @@ -59,6 +59,7 @@ class Migration(migrations.Migration): ], options={ 'abstract': False, + 'verbose_name': 'Sales Order Line Item', }, ), migrations.CreateModel( diff --git a/src/backend/InvenTree/order/migrations/0024_salesorderallocation.py b/src/backend/InvenTree/order/migrations/0024_salesorderallocation.py index ca8ed182d998..d3a4623b202b 100644 --- a/src/backend/InvenTree/order/migrations/0024_salesorderallocation.py +++ b/src/backend/InvenTree/order/migrations/0024_salesorderallocation.py @@ -22,5 +22,8 @@ class Migration(migrations.Migration): ('item', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='sales_order_allocation', to='stock.StockItem')), ('line', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='allocations', to='order.SalesOrderLineItem')), ], + options={ + 'verbose_name': 'Sales Order Allocation', + }, ), ] diff --git a/src/backend/InvenTree/order/migrations/0053_salesordershipment.py b/src/backend/InvenTree/order/migrations/0053_salesordershipment.py index 85ab90f46ad9..440b9fbeefaa 100644 --- a/src/backend/InvenTree/order/migrations/0053_salesordershipment.py +++ b/src/backend/InvenTree/order/migrations/0053_salesordershipment.py @@ -23,5 +23,8 @@ class Migration(migrations.Migration): ('checked_by', models.ForeignKey(blank=True, help_text='User who checked this shipment', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL, verbose_name='Checked By')), ('order', models.ForeignKey(help_text='Sales Order', on_delete=django.db.models.deletion.CASCADE, related_name='shipments', to='order.salesorder', verbose_name='Order')), ], + options={ + 'verbose_name': 'Sales Order Shipment', + }, ), ] diff --git a/src/backend/InvenTree/order/migrations/0064_purchaseorderextraline_salesorderextraline.py b/src/backend/InvenTree/order/migrations/0064_purchaseorderextraline_salesorderextraline.py index 1c3d2ff7435e..86784edf939d 100644 --- a/src/backend/InvenTree/order/migrations/0064_purchaseorderextraline_salesorderextraline.py +++ b/src/backend/InvenTree/order/migrations/0064_purchaseorderextraline_salesorderextraline.py @@ -86,6 +86,7 @@ class Migration(migrations.Migration): ], options={ 'abstract': False, + 'verbose_name': 'Sales Order Extra Line', }, ), migrations.CreateModel( @@ -103,6 +104,7 @@ class Migration(migrations.Migration): ], options={ 'abstract': False, + 'verbose_name': 'Purchase Order Extra Line', }, ), migrations.RunPython(convert_line_items, reverse_code=nunconvert_line_items), diff --git a/src/backend/InvenTree/order/migrations/0083_returnorderextraline.py b/src/backend/InvenTree/order/migrations/0083_returnorderextraline.py index ba1d8c2812db..dd6bf6976746 100644 --- a/src/backend/InvenTree/order/migrations/0083_returnorderextraline.py +++ b/src/backend/InvenTree/order/migrations/0083_returnorderextraline.py @@ -30,6 +30,7 @@ class Migration(migrations.Migration): ], options={ 'abstract': False, + 'verbose_name': 'Return Order Extra Line', }, ), ] diff --git a/src/backend/InvenTree/order/migrations/0085_auto_20230322_1056.py b/src/backend/InvenTree/order/migrations/0085_auto_20230322_1056.py index 9a5f4a652fa2..ea3ad7223dde 100644 --- a/src/backend/InvenTree/order/migrations/0085_auto_20230322_1056.py +++ b/src/backend/InvenTree/order/migrations/0085_auto_20230322_1056.py @@ -44,6 +44,7 @@ class Migration(migrations.Migration): ], options={ 'unique_together': {('order', 'item')}, + 'verbose_name': 'Return Order Line Item', }, ), ] diff --git a/src/backend/InvenTree/order/models.py b/src/backend/InvenTree/order/models.py index 077dd94bdad1..f4abb444cd54 100644 --- a/src/backend/InvenTree/order/models.py +++ b/src/backend/InvenTree/order/models.py @@ -1355,6 +1355,11 @@ class PurchaseOrderLineItem(OrderLineItem): order: Reference to a PurchaseOrder object """ + class Meta: + """Model meta options.""" + + verbose_name = _('Purchase Order Line Item') + # Filter for determining if a particular PurchaseOrderLineItem is overdue OVERDUE_FILTER = ( Q(received__lt=F('quantity')) @@ -1492,6 +1497,11 @@ class PurchaseOrderExtraLine(OrderExtraLine): price: The unit price for this OrderLine """ + class Meta: + """Model meta options.""" + + verbose_name = _('Purchase Order Extra Line') + @staticmethod def get_api_url(): """Return the API URL associated with the PurchaseOrderExtraLine model.""" @@ -1516,6 +1526,11 @@ class SalesOrderLineItem(OrderLineItem): shipped: The number of items which have actually shipped against this line item """ + class Meta: + """Model meta options.""" + + verbose_name = _('Sales Order Line Item') + # Filter for determining if a particular SalesOrderLineItem is overdue OVERDUE_FILTER = ( Q(shipped__lt=F('quantity')) @@ -1649,6 +1664,7 @@ class Meta: # Shipment reference must be unique for a given sales order unique_together = ['order', 'reference'] + verbose_name = _('Sales Order Shipment') @staticmethod def get_api_url(): @@ -1806,6 +1822,11 @@ class SalesOrderExtraLine(OrderExtraLine): price: The unit price for this OrderLine """ + class Meta: + """Model meta options.""" + + verbose_name = _('Sales Order Extra Line') + @staticmethod def get_api_url(): """Return the API URL associated with the SalesOrderExtraLine model.""" @@ -1830,6 +1851,11 @@ class SalesOrderAllocation(models.Model): quantity: Quantity to take from the StockItem """ + class Meta: + """Model meta options.""" + + verbose_name = _('Sales Order Allocation') + @staticmethod def get_api_url(): """Return the API URL associated with the SalesOrderAllocation model.""" @@ -2208,6 +2234,7 @@ class ReturnOrderLineItem(OrderLineItem): class Meta: """Metaclass options for this model.""" + verbose_name = _('Return Order Line Item') unique_together = [('order', 'item')] @staticmethod @@ -2270,6 +2297,11 @@ def received(self): class ReturnOrderExtraLine(OrderExtraLine): """Model for a single ExtraLine in a ReturnOrder.""" + class Meta: + """Metaclass options for this model.""" + + verbose_name = _('Return Order Extra Line') + @staticmethod def get_api_url(): """Return the API URL associated with the ReturnOrderExtraLine model.""" From 3a43efd8a62e2775dc6554af5efb63c20e4e0153 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 04:21:01 +0000 Subject: [PATCH 144/190] Cleanup import data table rendering --- src/backend/InvenTree/order/serializers.py | 20 +++++++-- .../importer/ImportDataSelector.tsx | 41 +++++++++++++++---- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/backend/InvenTree/order/serializers.py b/src/backend/InvenTree/order/serializers.py index 4c6a9bdd45e5..cc9fcbece586 100644 --- a/src/backend/InvenTree/order/serializers.py +++ b/src/backend/InvenTree/order/serializers.py @@ -167,6 +167,14 @@ def order_fields(extra_fields): ] + extra_fields +class AbstractLineItemSerializer: + """Abstract serializer for LineItem object.""" + + target_date = serializers.DateField( + required=False, allow_null=True, label=_('Target Date') + ) + + class AbstractExtraLineSerializer( DataImportExportSerializerMixin, serializers.Serializer ): @@ -351,7 +359,9 @@ def save(self): @register_importer() class PurchaseOrderLineItemSerializer( - DataImportExportSerializerMixin, InvenTreeModelSerializer + DataImportExportSerializerMixin, + AbstractLineItemSerializer, + InvenTreeModelSerializer, ): """Serializer class for the PurchaseOrderLineItem model.""" @@ -956,7 +966,9 @@ def __init__(self, *args, **kwargs): @register_importer() class SalesOrderLineItemSerializer( - DataImportExportSerializerMixin, InvenTreeModelSerializer + DataImportExportSerializerMixin, + AbstractLineItemSerializer, + InvenTreeModelSerializer, ): """Serializer for a SalesOrderLineItem object.""" @@ -1738,7 +1750,9 @@ def save(self): @register_importer() class ReturnOrderLineItemSerializer( - DataImportExportSerializerMixin, InvenTreeModelSerializer + DataImportExportSerializerMixin, + AbstractLineItemSerializer, + InvenTreeModelSerializer, ): """Serializer for a ReturnOrderLineItem object.""" diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 8171f34c2f6f..e1ba132f4ed4 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -65,17 +65,10 @@ function ImporterDataCell({ return ( - + {cellValue} -
- - - - - -
@@ -154,6 +147,23 @@ export default function ImporterDataSelector({ onFormSuccess: () => table.refreshTable() }); + const rowErrors = useCallback((row: any) => { + if (!row.errors) { + return []; + } + + let errors: string[] = []; + + for (const k of Object.keys(row.errors)) { + console.log('errors:', k, row.errors[k]); + row.errors[k].forEach((e: string) => { + errors.push(`${k}: ${e}`); + }); + } + + return errors; + }, []); + const columns: TableColumn[] = useMemo(() => { let columns: TableColumn[] = [ { @@ -168,7 +178,20 @@ export default function ImporterDataSelector({ {row.valid ? ( ) : ( - + + + + + + + {rowErrors(row).map((error: string) => ( + + {error} + + ))} + + + )} ); From 5106ae708ae735e6016999e691537dfbb8b3f1af Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 04:40:11 +0000 Subject: [PATCH 145/190] Pass session information through to cell renderer --- .../importer/ImportDataSelector.tsx | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index e1ba132f4ed4..e7af1f03f0b0 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -1,17 +1,6 @@ import { t } from '@lingui/macro'; -import { - ActionIcon, - Group, - HoverCard, - Stack, - Text, - Tooltip -} from '@mantine/core'; -import { - IconCircleCheck, - IconEdit, - IconExclamationCircle -} from '@tabler/icons-react'; +import { Group, HoverCard, Stack, Text } from '@mantine/core'; +import { IconCircleCheck, IconExclamationCircle } from '@tabler/icons-react'; import { useCallback, useMemo, useState } from 'react'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; @@ -30,10 +19,12 @@ import { RowDeleteAction, RowEditAction } from '../../tables/RowActions'; import { ApiFormFieldSet } from '../forms/fields/ApiFormField'; function ImporterDataCell({ + session, column, row, onEdit }: { + session: ImportSessionState; column: any; row: any; onEdit?: () => void; @@ -54,8 +45,10 @@ function ImporterDataCell({ }, [row.errors, column.field]); const cellValue = useMemo(() => { + // TODO: Render inline models, rather than raw PK values + return row.data ? row.data[column.field] : ''; - }, [row.data, column.field]); + }, [row.data, column.field, session.availableFields]); const cellValid: boolean = useMemo( () => cellErrors.length == 0, @@ -155,7 +148,6 @@ export default function ImporterDataSelector({ let errors: string[] = []; for (const k of Object.keys(row.errors)) { - console.log('errors:', k, row.errors[k]); row.errors[k].forEach((e: string) => { errors.push(`${k}: ${e}`); }); @@ -206,6 +198,7 @@ export default function ImporterDataSelector({ render: (row: any) => { return ( editCell(row, column)} From fecd7f3f92daeddf7c226ea92719c8c5f8ebb18d Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 04:43:19 +0000 Subject: [PATCH 146/190] add separate 'field_overrides' field --- .../importer/migrations/0001_initial.py | 32 +++++++++---------- src/backend/InvenTree/importer/models.py | 10 +++++- src/backend/InvenTree/importer/validators.py | 2 +- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/backend/InvenTree/importer/migrations/0001_initial.py b/src/backend/InvenTree/importer/migrations/0001_initial.py index 8cd2009e9c85..13440c66f725 100644 --- a/src/backend/InvenTree/importer/migrations/0001_initial.py +++ b/src/backend/InvenTree/importer/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 4.2.11 on 2024-03-22 01:09 +# Generated by Django 4.2.12 on 2024-06-30 04:42 from django.conf import settings import django.core.validators @@ -6,9 +6,6 @@ import django.db.models.deletion import importer.validators -from InvenTree.helpers import GetExportFormats -from importer.status_codes import DataImportStatusCode - class Migration(migrations.Migration): @@ -24,23 +21,15 @@ class Migration(migrations.Migration): fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('timestamp', models.DateTimeField(auto_now_add=True, verbose_name='Timestamp')), - ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=GetExportFormats()), importer.validators.validate_data_file], verbose_name='Data File')), + ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['csv', 'tsv', 'xls', 'xlsx', 'json', 'yaml']), importer.validators.validate_data_file], verbose_name='Data File')), ('columns', models.JSONField(blank=True, null=True, verbose_name='Columns')), ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), - ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL, help_text='Import status')), - ('field_defaults', models.JSONField(blank=True, null=True, verbose_name='Field Defaults', validators=[importer.validators.validate_field_defaults])), + ('status', models.PositiveIntegerField(choices=[(0, 'Initializing'), (10, 'Mapping Columns'), (20, 'Importing Data'), (30, 'Processing Data'), (40, 'Complete')], default=0, help_text='Import status')), + ('field_defaults', models.JSONField(blank=True, null=True, validators=[importer.validators.validate_field_defaults], verbose_name='Field Defaults')), + ('field_overrides', models.JSONField(blank=True, null=True, validators=[importer.validators.validate_field_defaults], verbose_name='Field Overrides')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], ), - migrations.CreateModel( - name='DataImportColumnMap', - fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('field', models.CharField(max_length=100, verbose_name='Field')), - ('column', models.CharField(blank=True, max_length=100, verbose_name='Column')), - ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='column_mappings', to='importer.dataimportsession', verbose_name='Import Session')), - ], - ), migrations.CreateModel( name='DataImportRow', fields=[ @@ -49,9 +38,18 @@ class Migration(migrations.Migration): ('row_data', models.JSONField(blank=True, null=True, verbose_name='Original row data')), ('data', models.JSONField(blank=True, null=True, verbose_name='Data')), ('errors', models.JSONField(blank=True, null=True, verbose_name='Errors')), - ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rows', to='importer.dataimportsession', verbose_name='Import Session')), ('valid', models.BooleanField(default=False, verbose_name='Valid')), ('complete', models.BooleanField(default=False, verbose_name='Complete')), + ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rows', to='importer.dataimportsession', verbose_name='Import Session')), + ], + ), + migrations.CreateModel( + name='DataImportColumnMap', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('field', models.CharField(max_length=100, verbose_name='Field')), + ('column', models.CharField(blank=True, max_length=100, verbose_name='Column')), + ('session', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='column_mappings', to='importer.dataimportsession', verbose_name='Import Session')), ], ), ] diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 6b1e203ea654..4e91c5b36f4a 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -32,7 +32,8 @@ class DataImportSession(models.Model): data_file: FileField for the data file to import status: IntegerField for the status of the import session user: ForeignKey to the User who initiated the import - field_defaults: JSONField for field overrides (e.g. custom field values) + field_defaults: JSONField for field default values + field_overrides: JSONField for field overrides """ @staticmethod @@ -93,6 +94,13 @@ def save(self, *args, **kwargs): validators=[importer.validators.validate_field_defaults], ) + field_overrides = models.JSONField( + blank=True, + null=True, + verbose_name=_('Field Overrides'), + validators=[importer.validators.validate_field_defaults], + ) + @property def field_mapping(self): """Construct a dict of field mappings for this import session. diff --git a/src/backend/InvenTree/importer/validators.py b/src/backend/InvenTree/importer/validators.py index ee50f4017387..74f056d306a7 100644 --- a/src/backend/InvenTree/importer/validators.py +++ b/src/backend/InvenTree/importer/validators.py @@ -46,4 +46,4 @@ def validate_field_defaults(value): return if type(value) is not dict: - raise ValidationError(_('Field defaults must be a dictionary')) + raise ValidationError(_('Value must be a valid dictionary object')) From de6b57fc03ad2d4f16533c295ab53564b4c41a74 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 04:43:59 +0000 Subject: [PATCH 147/190] Expose 'field_overrides' to API --- src/backend/InvenTree/importer/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/InvenTree/importer/serializers.py b/src/backend/InvenTree/importer/serializers.py index aab1d28859d7..222982c9d0ce 100644 --- a/src/backend/InvenTree/importer/serializers.py +++ b/src/backend/InvenTree/importer/serializers.py @@ -46,6 +46,7 @@ class Meta: 'columns', 'column_mappings', 'field_defaults', + 'field_overrides', 'row_count', 'completed_row_count', ] From 07c611a404e3fcb6912acec1d541532c74cb1656 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 05:27:26 +0000 Subject: [PATCH 148/190] Refactor import field selection --- src/backend/InvenTree/InvenTree/metadata.py | 6 +- .../migrations/0032_auto_20210403_1837.py | 6 +- src/backend/InvenTree/company/models.py | 6 +- src/backend/InvenTree/company/serializers.py | 2 + src/backend/InvenTree/importer/mixins.py | 23 ++++++-- src/backend/InvenTree/importer/models.py | 55 ++++++++++--------- src/backend/InvenTree/importer/operations.py | 38 ------------- .../components/forms/fields/ChoiceField.tsx | 1 + .../importer/ImportDataSelector.tsx | 4 +- .../importer/ImporterColumnSelector.tsx | 2 +- 10 files changed, 63 insertions(+), 80 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/metadata.py b/src/backend/InvenTree/InvenTree/metadata.py index aa3d2529cffc..88a7f9962e4a 100644 --- a/src/backend/InvenTree/InvenTree/metadata.py +++ b/src/backend/InvenTree/InvenTree/metadata.py @@ -319,8 +319,10 @@ def get_field_info(self, field): # Force non-nullable fields to read as "required" # (even if there is a default value!) - if not field.allow_null and not ( - hasattr(field, 'allow_blank') and field.allow_blank + if ( + 'required' not in field_info + and not field.allow_null + and not (hasattr(field, 'allow_blank') and field.allow_blank) ): field_info['required'] = True diff --git a/src/backend/InvenTree/company/migrations/0032_auto_20210403_1837.py b/src/backend/InvenTree/company/migrations/0032_auto_20210403_1837.py index d9c83d75ed1a..8b5f6fb89ff4 100644 --- a/src/backend/InvenTree/company/migrations/0032_auto_20210403_1837.py +++ b/src/backend/InvenTree/company/migrations/0032_auto_20210403_1837.py @@ -23,17 +23,17 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='company', name='is_customer', - field=models.BooleanField(default=False, help_text='Do you sell items to this company?', verbose_name='is customer'), + field=models.BooleanField(default=False, help_text='Do you sell items to this company?', verbose_name='Is customer'), ), migrations.AlterField( model_name='company', name='is_manufacturer', - field=models.BooleanField(default=False, help_text='Does this company manufacture parts?', verbose_name='is manufacturer'), + field=models.BooleanField(default=False, help_text='Does this company manufacture parts?', verbose_name='Is manufacturer'), ), migrations.AlterField( model_name='company', name='is_supplier', - field=models.BooleanField(default=True, help_text='Do you purchase items from this company?', verbose_name='is supplier'), + field=models.BooleanField(default=True, help_text='Do you purchase items from this company?', verbose_name='Is supplier'), ), migrations.AlterField( model_name='company', diff --git a/src/backend/InvenTree/company/models.py b/src/backend/InvenTree/company/models.py index 3f5b5298b3b0..58b093856896 100644 --- a/src/backend/InvenTree/company/models.py +++ b/src/backend/InvenTree/company/models.py @@ -165,19 +165,19 @@ def get_api_url(): is_customer = models.BooleanField( default=False, - verbose_name=_('is customer'), + verbose_name=_('Is customer'), help_text=_('Do you sell items to this company?'), ) is_supplier = models.BooleanField( default=True, - verbose_name=_('is supplier'), + verbose_name=_('Is supplier'), help_text=_('Do you purchase items from this company?'), ) is_manufacturer = models.BooleanField( default=False, - verbose_name=_('is manufacturer'), + verbose_name=_('Is manufacturer'), help_text=_('Does this company manufacture parts?'), ) diff --git a/src/backend/InvenTree/company/serializers.py b/src/backend/InvenTree/company/serializers.py index a0fba43fa640..a32cd5e388a5 100644 --- a/src/backend/InvenTree/company/serializers.py +++ b/src/backend/InvenTree/company/serializers.py @@ -114,6 +114,8 @@ class CompanySerializer( export_exclude_fields = ['url', 'primary_address'] + import_exclude_fields = ['image'] + class Meta: """Metaclass options.""" diff --git a/src/backend/InvenTree/importer/mixins.py b/src/backend/InvenTree/importer/mixins.py index ea1a25a30aa7..a5a4624a39f7 100644 --- a/src/backend/InvenTree/importer/mixins.py +++ b/src/backend/InvenTree/importer/mixins.py @@ -4,7 +4,7 @@ from django.utils.translation import gettext_lazy as _ import tablib -from rest_framework import serializers +from rest_framework import fields, serializers import importer.operations from InvenTree.helpers import DownloadFile, GetExportFormats, current_date @@ -35,9 +35,18 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if importing: - # Exclude fields which are not required for data import + # Exclude any fields which are not able to be imported + importable_field_names = list(self.get_importable_fields().keys()) + field_names = list(self.fields.keys()) + + for field in field_names: + if field not in importable_field_names: + self.fields.pop(field, None) + + # Exclude fields which are excluded for data import for field in self.get_import_exclude_fields(**kwargs): self.fields.pop(field, None) + else: # Exclude fields which are only used for data import for field in self.get_import_only_fields(**kwargs): @@ -49,7 +58,7 @@ def get_importable_fields(self) -> dict: Returns: dict: A dictionary of field names and field objects """ - fields = {} + importable_fields = {} for name, field in self.fields.items(): # Skip read-only fields @@ -60,9 +69,13 @@ def get_importable_fields(self) -> dict: if issubclass(field.__class__, serializers.Serializer): continue - fields[name] = field + # Skip file fields + if issubclass(field.__class__, fields.FileField): + continue - return fields + importable_fields[name] = field + + return importable_fields class DataExportSerializerMixin: diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 4e91c5b36f4a..2de079d82dd0 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -121,21 +121,6 @@ def serializer_class(self): return supported_models().get(self.model_type, None) - def serializer_fields(self, required=None, read_only=False, write_only=None): - """Return the writeable serializers fields for this importer. - - Arguments: - required: If True, only return required fields - """ - from importer.operations import get_fields - - return get_fields( - self.serializer_class, - required=required, - read_only=read_only, - write_only=write_only, - ) - def extract_columns(self): """Run initial column extraction and mapping. @@ -147,7 +132,7 @@ def extract_columns(self): # Extract list of column names from the file self.columns = importer.operations.extract_column_names(self.data_file) - serializer_fields = self.serializer_fields(read_only=False) + serializer_fields = self.available_fields() # Remove any existing mappings self.column_mappings.all().delete() @@ -159,8 +144,11 @@ def extract_columns(self): # Create a default mapping for each available field in the database for field, field_def in serializer_fields.items(): # Generate a list of possible column names for this field - field_options = [field, getattr(field_def, 'label', field)] - + field_options = [ + field, + field_def.get('label', None), + field_def.get('help_text', None), + ] column_name = '' for column in self.columns: @@ -195,12 +183,18 @@ def accept_mapping(self): - Trigger the data import process """ # First, we need to ensure that all the *required* columns have been mapped - required_fields = self.serializer_fields(required=True).keys() + required_fields = self.required_fields() + + field_overrides = self.field_overrides or {} field_defaults = self.field_defaults or {} missing_fields = [] - for field in required_fields: + for field in required_fields.keys(): + # An override value exists + if field in field_overrides: + continue + # A default value exists if field in field_defaults: continue @@ -299,13 +293,20 @@ def available_fields(self): else: fields = {} - # Remove any read-only fields (they are of no use here) - for key in list(fields.keys()): - if fields[key].get('read_only', False): - del fields[key] - return fields + def required_fields(self): + """Returns information on which fields are *required* for import.""" + fields = self.available_fields() + + required = {} + + for field, info in fields.items(): + if info.get('required', False): + required[field] = info + + return required + class DataImportColumnMap(models.Model): """Database model representing a mapping between a file column and serializer field. @@ -367,7 +368,7 @@ def clean(self): 'field': _('Field does not exist in the target model') }) - if field_def.read_only: + if field_def.get('read_only', False): raise DjangoValidationError({'field': _('Selected field is read-only')}) session = models.ForeignKey( @@ -384,7 +385,7 @@ def clean(self): @property def field_definition(self): """Return the field definition associated with this column mapping.""" - fields = self.session.serializer_fields(read_only=None) + fields = self.session.available_fields() return fields.get(self.field, None) @property diff --git a/src/backend/InvenTree/importer/operations.py b/src/backend/InvenTree/importer/operations.py index e077f9e67ee0..f0c07774574d 100644 --- a/src/backend/InvenTree/importer/operations.py +++ b/src/backend/InvenTree/importer/operations.py @@ -88,44 +88,6 @@ def extract_rows(data_file) -> list: return rows -def get_fields( - serializer_class, - write_only=None, - read_only=None, - required=None, - exporting=False, - importing=True, -): - """Extract the field names from a serializer class. - - Arguments: - serializer_class: Serializer - write_only: Filter fields based on write_only attribute - read_only: Filter fields based on read_only attribute - required: Filter fields based on required attribute - """ - if not serializer_class: - return {} - - serializer = serializer_class(exporting=exporting, importing=importing) - - fields = {} - - for field_name, field in serializer.fields.items(): - if read_only is not None and getattr(field, 'read_only', None) != read_only: - continue - - if write_only is not None and getattr(field, 'write_only', None) != write_only: - continue - - if required is not None and getattr(field, 'required', None) != required: - continue - - fields[field_name] = field - - return fields - - def get_field_label(field) -> str: """Return the label for a field in a serializer class. diff --git a/src/frontend/src/components/forms/fields/ChoiceField.tsx b/src/frontend/src/components/forms/fields/ChoiceField.tsx index 2f47c727186e..7407edf4b088 100644 --- a/src/frontend/src/components/forms/fields/ChoiceField.tsx +++ b/src/frontend/src/components/forms/fields/ChoiceField.tsx @@ -65,6 +65,7 @@ export function ChoiceField({ disabled={definition.disabled} leftSection={definition.icon} comboboxProps={{ withinPortal: true }} + searchable /> ); } diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index e7af1f03f0b0..279fabe45234 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -60,7 +60,9 @@ function ImporterDataCell({ - {cellValue} + + {cellValue} + diff --git a/src/frontend/src/components/importer/ImporterColumnSelector.tsx b/src/frontend/src/components/importer/ImporterColumnSelector.tsx index b64333c875d6..370e8da1a052 100644 --- a/src/frontend/src/components/importer/ImporterColumnSelector.tsx +++ b/src/frontend/src/components/importer/ImporterColumnSelector.tsx @@ -96,7 +96,7 @@ export default function ImporterColumnSelector({ }; }) ]; - }, [session.columnMappings]); + }, [session.availableColumns]); return ( From 1392805cf05562fd72db0dfac90cbfe666572941 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 05:31:00 +0000 Subject: [PATCH 149/190] Use override data if provided --- src/backend/InvenTree/importer/models.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 2de079d82dd0..fd2dc0883c7a 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -448,11 +448,22 @@ def extract_data(self, field_mapping: dict = None): if not field_mapping: field_mapping = self.session.field_mapping + override_values = self.session.field_overrides or {} + default_values = self.session.field_defaults or {} + data = {} # We have mapped column (file) to field (serializer) already for field, col in field_mapping.items(): - data[field] = self.row_data.get(col, None) + # If an override value exists, use that + if field in override_values: + value = override_values[field] + else: + value = self.row_data.get(col, None) + if value is None and field in default_values: + value = default_values[field] + + data[field] = value self.data = data self.save() @@ -460,11 +471,13 @@ def extract_data(self, field_mapping: dict = None): def serializer_data(self): """Construct data object to be sent to the serializer. - Note that we also use the "default" values provided by the import session + - If available, we use the "default" values provided by the import session + - If available, we use the "override" values provided by the import session """ session_defaults = self.session.field_defaults or {} + session_overrides = self.session.field_overrides or {} - return {**session_defaults, **self.data} + return {**session_defaults, **self.data, **session_overrides} def construct_serializer(self): """Construct a serializer object for this row.""" From 5afed6f951d80e7597ac47650d9bec0055a5ce23 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 05:43:57 +0000 Subject: [PATCH 150/190] Fix data extraction - Ignore columns which are not mapped --- .../InvenTree/InvenTree/serializers.py | 2 +- src/backend/InvenTree/importer/models.py | 22 ++++++++++++++----- .../importer/ImportDataSelector.tsx | 8 ++++++- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/serializers.py b/src/backend/InvenTree/InvenTree/serializers.py index 829216089f09..406368511559 100644 --- a/src/backend/InvenTree/InvenTree/serializers.py +++ b/src/backend/InvenTree/InvenTree/serializers.py @@ -856,7 +856,7 @@ def skip_create_fields(self): remote_image = serializers.URLField( required=False, - allow_blank=False, + allow_blank=True, write_only=True, label=_('Remote Image'), help_text=_('URL of remote image file'), diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index fd2dc0883c7a..67a44823f172 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -457,11 +457,18 @@ def extract_data(self, field_mapping: dict = None): for field, col in field_mapping.items(): # If an override value exists, use that if field in override_values: - value = override_values[field] - else: - value = self.row_data.get(col, None) - if value is None and field in default_values: - value = default_values[field] + data[field] = override_values[field] + continue + + # If this field is *not* mapped to any column, skip + if not col: + continue + + value = self.row_data.get(col, None) + + # Use the default value, if provided + if value in [None, ''] and field in default_values: + value = default_values[field] data[field] = value @@ -477,7 +484,10 @@ def serializer_data(self): session_defaults = self.session.field_defaults or {} session_overrides = self.session.field_overrides or {} - return {**session_defaults, **self.data, **session_overrides} + # Construct data + data = {**session_defaults, **self.data, **session_overrides} + + return data def construct_serializer(self): """Construct a serializer object for this row.""" diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 279fabe45234..e70f4f653c4c 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -47,7 +47,13 @@ function ImporterDataCell({ const cellValue = useMemo(() => { // TODO: Render inline models, rather than raw PK values - return row.data ? row.data[column.field] : ''; + let value = row.data ? row.data[column.field] ?? '' : ''; + + if (!value) { + value = '-'; + } + + return value; }, [row.data, column.field, session.availableFields]); const cellValid: boolean = useMemo( From cd5302c143a011c3f382065588dc7eb96a9d3a74 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 06:02:32 +0000 Subject: [PATCH 151/190] Fix fields.pop - Provide 'None' argument --- src/backend/InvenTree/InvenTree/forms.py | 2 +- src/backend/InvenTree/build/serializers.py | 10 +++--- src/backend/InvenTree/company/serializers.py | 22 ++++++------ src/backend/InvenTree/importer/api.py | 5 +-- src/backend/InvenTree/order/serializers.py | 38 ++++++++++---------- src/backend/InvenTree/part/serializers.py | 36 +++++++++---------- src/backend/InvenTree/stock/serializers.py | 20 +++++------ 7 files changed, 67 insertions(+), 66 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/forms.py b/src/backend/InvenTree/InvenTree/forms.py index b4a992e1d996..cffb987e1f47 100644 --- a/src/backend/InvenTree/InvenTree/forms.py +++ b/src/backend/InvenTree/InvenTree/forms.py @@ -190,7 +190,7 @@ def __init__(self, *args, **kwargs): # check for two password fields if not get_global_setting('LOGIN_SIGNUP_PWD_TWICE'): - self.fields.pop('password2') + self.fields.pop('password2', None) # reorder fields set_form_field_order( diff --git a/src/backend/InvenTree/build/serializers.py b/src/backend/InvenTree/build/serializers.py index 8e2bf2ccff58..1e8d308b955f 100644 --- a/src/backend/InvenTree/build/serializers.py +++ b/src/backend/InvenTree/build/serializers.py @@ -125,7 +125,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if part_detail is not True: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) reference = serializers.CharField(required=True) @@ -1117,16 +1117,16 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not part_detail: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) if not location_detail: - self.fields.pop('location_detail') + self.fields.pop('location_detail', None) if not stock_detail: - self.fields.pop('stock_item_detail') + self.fields.pop('stock_item_detail', None) if not build_detail: - self.fields.pop('build_detail') + self.fields.pop('build_detail', None) class BuildLineSerializer(DataImportExportSerializerMixin, InvenTreeModelSerializer): diff --git a/src/backend/InvenTree/company/serializers.py b/src/backend/InvenTree/company/serializers.py index a32cd5e388a5..5ef5f248fff6 100644 --- a/src/backend/InvenTree/company/serializers.py +++ b/src/backend/InvenTree/company/serializers.py @@ -246,13 +246,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if part_detail is not True: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) if manufacturer_detail is not True: - self.fields.pop('manufacturer_detail') + self.fields.pop('manufacturer_detail', None) if prettify is not True: - self.fields.pop('pretty_name') + self.fields.pop('pretty_name', None) part_detail = PartBriefSerializer(source='part', many=False, read_only=True) @@ -294,7 +294,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not man_detail: - self.fields.pop('manufacturer_part_detail') + self.fields.pop('manufacturer_part_detail', None) manufacturer_part_detail = ManufacturerPartSerializer( source='manufacturer_part', many=False, read_only=True @@ -368,17 +368,17 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if part_detail is not True: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) if supplier_detail is not True: - self.fields.pop('supplier_detail') + self.fields.pop('supplier_detail', None) if manufacturer_detail is not True: - self.fields.pop('manufacturer_detail') - self.fields.pop('manufacturer_part_detail') + self.fields.pop('manufacturer_detail', None) + self.fields.pop('manufacturer_part_detail', None) if prettify is not True: - self.fields.pop('pretty_name') + self.fields.pop('pretty_name', None) # Annotated field showing total in-stock quantity in_stock = serializers.FloatField(read_only=True, label=_('In Stock')) @@ -492,10 +492,10 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not supplier_detail: - self.fields.pop('supplier_detail') + self.fields.pop('supplier_detail', None) if not part_detail: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) quantity = InvenTreeDecimalField() diff --git a/src/backend/InvenTree/importer/api.py b/src/backend/InvenTree/importer/api.py index 25aa5e7a6c5d..7472e30025c5 100644 --- a/src/backend/InvenTree/importer/api.py +++ b/src/backend/InvenTree/importer/api.py @@ -11,6 +11,7 @@ import importer.models import importer.registry import importer.serializers +from InvenTree.api import BulkDeleteMixin from InvenTree.filters import SEARCH_ORDER_FILTER from InvenTree.mixins import ( ListAPI, @@ -42,7 +43,7 @@ def get(self, request): return Response(models) -class DataImportSessionList(ListCreateAPI): +class DataImportSessionList(BulkDeleteMixin, ListCreateAPI): """API endpoint for accessing a list of DataImportSession objects.""" queryset = importer.models.DataImportSession.objects.all() @@ -98,7 +99,7 @@ class DataImportColumnMappingDetail(RetrieveUpdateAPI): serializer_class = importer.serializers.DataImportColumnMapSerializer -class DataImportRowList(ListAPI): +class DataImportRowList(BulkDeleteMixin, ListAPI): """API endpoint for accessing a list of DataImportRow objects.""" queryset = importer.models.DataImportRow.objects.all() diff --git a/src/backend/InvenTree/order/serializers.py b/src/backend/InvenTree/order/serializers.py index cc9fcbece586..40bc800f932c 100644 --- a/src/backend/InvenTree/order/serializers.py +++ b/src/backend/InvenTree/order/serializers.py @@ -187,7 +187,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if order_detail is not True: - self.fields.pop('order_detail') + self.fields.pop('order_detail', None) quantity = serializers.FloatField() @@ -249,7 +249,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if supplier_detail is not True: - self.fields.pop('supplier_detail') + self.fields.pop('supplier_detail', None) @staticmethod def annotate_queryset(queryset): @@ -407,11 +407,11 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if part_detail is not True: - self.fields.pop('part_detail') - self.fields.pop('supplier_part_detail') + self.fields.pop('part_detail', None) + self.fields.pop('supplier_part_detail', None) if order_detail is not True: - self.fields.pop('order_detail') + self.fields.pop('order_detail', None) def skip_create_fields(self): """Return a list of fields to skip when creating a new object.""" @@ -835,7 +835,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if customer_detail is not True: - self.fields.pop('customer_detail') + self.fields.pop('customer_detail', None) @staticmethod def annotate_queryset(queryset): @@ -922,19 +922,19 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not order_detail: - self.fields.pop('order_detail') + self.fields.pop('order_detail', None) if not part_detail: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) if not item_detail: - self.fields.pop('item_detail') + self.fields.pop('item_detail', None) if not location_detail: - self.fields.pop('location_detail') + self.fields.pop('location_detail', None) if not customer_detail: - self.fields.pop('customer_detail') + self.fields.pop('customer_detail', None) part = serializers.PrimaryKeyRelatedField(source='item.part', read_only=True) order = serializers.PrimaryKeyRelatedField( @@ -1012,16 +1012,16 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if part_detail is not True: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) if order_detail is not True: - self.fields.pop('order_detail') + self.fields.pop('order_detail', None) if allocations is not True: - self.fields.pop('allocations') + self.fields.pop('allocations', None) if customer_detail is not True: - self.fields.pop('customer_detail') + self.fields.pop('customer_detail', None) @staticmethod def annotate_queryset(queryset): @@ -1597,7 +1597,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if customer_detail is not True: - self.fields.pop('customer_detail') + self.fields.pop('customer_detail', None) @staticmethod def annotate_queryset(queryset): @@ -1788,13 +1788,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not order_detail: - self.fields.pop('order_detail') + self.fields.pop('order_detail', None) if not item_detail: - self.fields.pop('item_detail') + self.fields.pop('item_detail', None) if not part_detail: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) order_detail = ReturnOrderSerializer(source='order', many=False, read_only=True) item_detail = stock.serializers.StockItemSerializer( diff --git a/src/backend/InvenTree/part/serializers.py b/src/backend/InvenTree/part/serializers.py index ba7df895cf32..485db6efbd7a 100644 --- a/src/backend/InvenTree/part/serializers.py +++ b/src/backend/InvenTree/part/serializers.py @@ -96,7 +96,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not path_detail: - self.fields.pop('path') + self.fields.pop('path', None) def get_starred(self, category) -> bool: """Return True if the category is directly "starred" by the current user.""" @@ -326,8 +326,8 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not pricing: - self.fields.pop('pricing_min') - self.fields.pop('pricing_max') + self.fields.pop('pricing_min', None) + self.fields.pop('pricing_max', None) category_default_location = serializers.IntegerField(read_only=True) @@ -374,10 +374,10 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not part_detail: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) if not template_detail: - self.fields.pop('template_detail') + self.fields.pop('template_detail', None) part_detail = PartBriefSerializer(source='part', many=False, read_only=True) template_detail = PartParameterTemplateSerializer( @@ -688,13 +688,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not category_detail: - self.fields.pop('category_detail') + self.fields.pop('category_detail', None) if not parameters: - self.fields.pop('parameters') + self.fields.pop('parameters', None) if not path_detail: - self.fields.pop('category_path') + self.fields.pop('category_path', None) if not create: # These fields are only used for the LIST API endpoint @@ -705,9 +705,9 @@ def __init__(self, *args, **kwargs): self.fields.pop(f) if not pricing: - self.fields.pop('pricing_min') - self.fields.pop('pricing_max') - self.fields.pop('pricing_updated') + self.fields.pop('pricing_min', None) + self.fields.pop('pricing_max', None) + self.fields.pop('pricing_updated', None) def get_api_url(self): """Return the API url associated with this serializer.""" @@ -1484,17 +1484,17 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not part_detail: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) if not sub_part_detail: - self.fields.pop('sub_part_detail') + self.fields.pop('sub_part_detail', None) if not pricing: - self.fields.pop('pricing_min') - self.fields.pop('pricing_max') - self.fields.pop('pricing_min_total') - self.fields.pop('pricing_max_total') - self.fields.pop('pricing_updated') + self.fields.pop('pricing_min', None) + self.fields.pop('pricing_max', None) + self.fields.pop('pricing_min_total', None) + self.fields.pop('pricing_max_total', None) + self.fields.pop('pricing_updated', None) quantity = InvenTree.serializers.InvenTreeDecimalField(required=True) diff --git a/src/backend/InvenTree/stock/serializers.py b/src/backend/InvenTree/stock/serializers.py index fd3390e828ee..cce574c12cfa 100644 --- a/src/backend/InvenTree/stock/serializers.py +++ b/src/backend/InvenTree/stock/serializers.py @@ -217,10 +217,10 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if user_detail is not True: - self.fields.pop('user_detail') + self.fields.pop('user_detail', None) if template_detail is not True: - self.fields.pop('template_detail') + self.fields.pop('template_detail', None) user_detail = InvenTree.serializers.UserSerializer(source='user', read_only=True) @@ -412,19 +412,19 @@ def __init__(self, *args, **kwargs): super(StockItemSerializer, self).__init__(*args, **kwargs) if not part_detail: - self.fields.pop('part_detail') + self.fields.pop('part_detail', None) if not location_detail: - self.fields.pop('location_detail') + self.fields.pop('location_detail', None) if not supplier_part_detail: - self.fields.pop('supplier_part_detail') + self.fields.pop('supplier_part_detail', None) if not tests: - self.fields.pop('tests') + self.fields.pop('tests', None) if not path_detail: - self.fields.pop('location_path') + self.fields.pop('location_path', None) part = serializers.PrimaryKeyRelatedField( queryset=part_models.Part.objects.all(), @@ -1080,7 +1080,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if not path_detail: - self.fields.pop('path') + self.fields.pop('path', None) @staticmethod def annotate_queryset(queryset): @@ -1158,10 +1158,10 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) if item_detail is not True: - self.fields.pop('item_detail') + self.fields.pop('item_detail', None) if user_detail is not True: - self.fields.pop('user_detail') + self.fields.pop('user_detail', None) label = serializers.CharField(read_only=True) From 25d09afec962c81d9af81e02eb0d8463ee9b2b0d Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 06:02:51 +0000 Subject: [PATCH 152/190] Update import data rendering --- .../importer/ImportDataSelector.tsx | 35 ++++++++++++++++--- .../tables/settings/ImportSessionTable.tsx | 2 ++ 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index e70f4f653c4c..aa798ba2f2b1 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -1,7 +1,11 @@ import { t } from '@lingui/macro'; import { Group, HoverCard, Stack, Text } from '@mantine/core'; -import { IconCircleCheck, IconExclamationCircle } from '@tabler/icons-react'; -import { useCallback, useMemo, useState } from 'react'; +import { + IconCircleCheck, + IconExclamationCircle, + IconSquareArrowRight +} from '@tabler/icons-react'; +import { ReactNode, useCallback, useMemo, useState } from 'react'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { cancelEvent } from '../../functions/events'; @@ -16,6 +20,7 @@ import { TableColumn } from '../../tables/Column'; import { TableFilter } from '../../tables/Filter'; import { InvenTreeTable } from '../../tables/InvenTreeTable'; import { RowDeleteAction, RowEditAction } from '../../tables/RowActions'; +import { YesNoButton } from '../buttons/YesNoButton'; import { ApiFormFieldSet } from '../forms/fields/ApiFormField'; function ImporterDataCell({ @@ -44,9 +49,20 @@ function ImporterDataCell({ return row?.errors[column.field] ?? []; }, [row.errors, column.field]); - const cellValue = useMemo(() => { + const cellValue: ReactNode = useMemo(() => { // TODO: Render inline models, rather than raw PK values + let field_def = session.availableFields[column.field]; + + switch (field_def?.type) { + case 'boolean': + return ( + + ); + default: + break; + } + let value = row.data ? row.data[column.field] ?? '' : ''; if (!value) { @@ -184,8 +200,9 @@ export default function ImporterDataSelector({ + {t`Row contains errors`}: {rowErrors(row).map((error: string) => ( - + {error} ))} @@ -223,7 +240,14 @@ export default function ImporterDataSelector({ const rowActions = useCallback( (record: any) => { return [ + { + title: t`Accept`, + icon: , + color: 'green', + hidden: record.complete + }, RowEditAction({ + hidden: record.complete, onClick: () => { setSelectedRow(record); setSelectedFieldNames( @@ -277,7 +301,8 @@ export default function ImporterDataSelector({ tableFilters: filters, enableColumnSwitching: true, enableColumnCaching: false, - enableSelection: true + enableSelection: true, + enableBulkDelete: true }} /> diff --git a/src/frontend/src/tables/settings/ImportSessionTable.tsx b/src/frontend/src/tables/settings/ImportSessionTable.tsx index 5326acb735c3..47bbd364bdab 100644 --- a/src/frontend/src/tables/settings/ImportSessionTable.tsx +++ b/src/frontend/src/tables/settings/ImportSessionTable.tsx @@ -158,6 +158,8 @@ export default function ImportSesssionTable() { rowActions: rowActions, tableActions: tableActions, tableFilters: tableFilters, + enableBulkDelete: true, + enableSelection: true, onRowClick: (record: any) => { setSelectedSession(record.pk); setOpened(true); From 7f5e036aea4406c897b0b0bf596a9373fe4812a5 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 06:12:49 +0000 Subject: [PATCH 153/190] Handle missing / empty column names when importing data --- src/backend/InvenTree/importer/models.py | 5 +++++ src/backend/InvenTree/importer/operations.py | 12 +++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 67a44823f172..1431126102db 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -152,6 +152,10 @@ def extract_columns(self): column_name = '' for column in self.columns: + # No title provided for the column + if not column: + continue + # Ignore if we have already matched this column to a field if column in matched_columns: continue @@ -288,6 +292,7 @@ def available_fields(self): from InvenTree.metadata import InvenTreeMetadata metadata = InvenTreeMetadata() + if serializer := self.serializer_class: fields = metadata.get_serializer_info(serializer(data={}, importing=True)) else: diff --git a/src/backend/InvenTree/importer/operations.py b/src/backend/InvenTree/importer/operations.py index f0c07774574d..7b9806d07b56 100644 --- a/src/backend/InvenTree/importer/operations.py +++ b/src/backend/InvenTree/importer/operations.py @@ -68,7 +68,17 @@ def extract_column_names(data_file) -> list: ValidationError: If the data file is not in a valid format """ data = load_data_file(data_file) - return data.headers + + headers = [] + + for idx, header in enumerate(data.headers): + if header: + headers.append(header) + else: + # If the header is empty, generate a default header + headers.append(f'Column {idx + 1}') + + return headers def extract_rows(data_file) -> list: From a8aafbba97ea8813791fbc487ea5483081b109e5 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 06:23:29 +0000 Subject: [PATCH 154/190] Bug fixin' --- src/backend/InvenTree/InvenTree/metadata.py | 3 +-- src/backend/InvenTree/importer/models.py | 9 +++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/metadata.py b/src/backend/InvenTree/InvenTree/metadata.py index 88a7f9962e4a..c810bbdbe4cf 100644 --- a/src/backend/InvenTree/InvenTree/metadata.py +++ b/src/backend/InvenTree/InvenTree/metadata.py @@ -169,8 +169,7 @@ def get_serializer_info(self, serializer): # Already know about this one continue - if hasattr(serializer, field_name): - field = getattr(serializer, field_name) + if field := getattr(serializer, field_name, None): serializer_info[field_name] = self.get_field_info(field) model_class = None diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 1431126102db..3ad80bfe3eff 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -146,8 +146,8 @@ def extract_columns(self): # Generate a list of possible column names for this field field_options = [ field, - field_def.get('label', None), - field_def.get('help_text', None), + field_def.get('label', field), + field_def.get('help_text', field), ] column_name = '' @@ -293,8 +293,9 @@ def available_fields(self): metadata = InvenTreeMetadata() - if serializer := self.serializer_class: - fields = metadata.get_serializer_info(serializer(data={}, importing=True)) + if serializer_class := self.serializer_class: + serializer = serializer_class(data={}, importing=True) + fields = metadata.get_serializer_info(serializer) else: fields = {} From 72314e213dd558473140d6d2890f35157be1eb66 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 06:28:07 +0000 Subject: [PATCH 155/190] Update hook --- src/frontend/src/hooks/UseImportSession.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/src/hooks/UseImportSession.tsx b/src/frontend/src/hooks/UseImportSession.tsx index 7da22ea65514..23f20a74fdfd 100644 --- a/src/frontend/src/hooks/UseImportSession.tsx +++ b/src/frontend/src/hooks/UseImportSession.tsx @@ -67,7 +67,7 @@ export function useImportSession({ ); return cols; - }, [sessionData]); + }, [sessionData.columns]); const columnMappings: any[] = useMemo(() => { let mapping = From ff48181a768ac2dafc481f9249ebbd2b5fc1e8ff Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sun, 30 Jun 2024 06:55:21 +0000 Subject: [PATCH 156/190] Adds button to upload data straight to table --- src/backend/InvenTree/stock/serializers.py | 4 ++++ src/frontend/src/components/buttons/ActionButton.tsx | 2 +- src/frontend/src/components/items/ActionDropdown.tsx | 2 +- src/frontend/src/tables/InvenTreeTable.tsx | 5 +++++ src/frontend/src/tables/UploadAction.tsx | 12 ++++++++++++ 5 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 src/frontend/src/tables/UploadAction.tsx diff --git a/src/backend/InvenTree/stock/serializers.py b/src/backend/InvenTree/stock/serializers.py index cce574c12cfa..a060bbc6b6cf 100644 --- a/src/backend/InvenTree/stock/serializers.py +++ b/src/backend/InvenTree/stock/serializers.py @@ -333,6 +333,8 @@ class StockItemSerializer( export_exclude_fields = ['tracking_items'] + import_exclude_fields = ['use_pack_size', 'tags'] + class Meta: """Metaclass options.""" @@ -1045,6 +1047,8 @@ class LocationSerializer( ): """Detailed information about a stock location.""" + import_exclude_fields = ['tags'] + class Meta: """Metaclass options.""" diff --git a/src/frontend/src/components/buttons/ActionButton.tsx b/src/frontend/src/components/buttons/ActionButton.tsx index 089cb9899506..dce1209dc145 100644 --- a/src/frontend/src/components/buttons/ActionButton.tsx +++ b/src/frontend/src/components/buttons/ActionButton.tsx @@ -43,7 +43,7 @@ export function ActionButton(props: ActionButtonProps) { props.tooltip ?? props.text ?? '' )}`} onClick={props.onClick ?? notYetImplemented} - variant={props.variant ?? 'light'} + variant={props.variant ?? 'transparent'} > {props.icon} diff --git a/src/frontend/src/components/items/ActionDropdown.tsx b/src/frontend/src/components/items/ActionDropdown.tsx index 73777a9c56db..5515858e0c76 100644 --- a/src/frontend/src/components/items/ActionDropdown.tsx +++ b/src/frontend/src/components/items/ActionDropdown.tsx @@ -66,7 +66,7 @@ export function ActionDropdown({ diff --git a/src/frontend/src/tables/InvenTreeTable.tsx b/src/frontend/src/tables/InvenTreeTable.tsx index e955b7721ffc..2bad26176f8f 100644 --- a/src/frontend/src/tables/InvenTreeTable.tsx +++ b/src/frontend/src/tables/InvenTreeTable.tsx @@ -54,6 +54,7 @@ import { TableFilter } from './Filter'; import { FilterSelectDrawer } from './FilterSelectDrawer'; import { RowAction, RowActions } from './RowActions'; import { TableSearchInput } from './Search'; +import { UploadAction } from './UploadAction'; const defaultPageSize: number = 25; @@ -66,6 +67,7 @@ const defaultPageSize: number = 25; * @param noRecordsText : string - Text to display when no records are found * @param enableBulkDelete : boolean - Enable bulk deletion of records * @param enableDownload : boolean - Enable download actions + * @param enableUpload : boolean - Enable upload actions * @param enableFilters : boolean - Enable filter actions * @param enableSelection : boolean - Enable row selection * @param enableSearch : boolean - Enable search actions @@ -91,6 +93,7 @@ export type InvenTreeTableProps = { noRecordsText?: string; enableBulkDelete?: boolean; enableDownload?: boolean; + enableUpload?: boolean; enableFilters?: boolean; enableSelection?: boolean; enableSearch?: boolean; @@ -122,6 +125,7 @@ const defaultInvenTreeTableProps: InvenTreeTableProps = { params: {}, noRecordsText: t`No records found`, enableDownload: false, + enableUpload: false, enableLabels: false, enableReports: false, enableFilters: true, @@ -603,6 +607,7 @@ export function InvenTreeTable({ downloadCallback={downloadData} /> )} + {tableProps.enableUpload && } + } tooltip={t`Upload Data`} /> + + ); +} From 4e2008bdc3228d417c02f308ab530c52dcee60fe Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Tue, 2 Jul 2024 13:04:22 +0000 Subject: [PATCH 157/190] Cache "available_fields" - Reduces API access time by 85% --- src/backend/InvenTree/importer/models.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 3ad80bfe3eff..9fe5880825c1 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -288,7 +288,12 @@ def available_fields(self): - This method is designed to be introspected by the frontend, for rendering the various fields. - We make use of the InvenTree.metadata module to provide extra information about the fields. + + Note that we cache these fields, as they are expensive to compute. """ + if fields := getattr(self, '_available_fields', None): + return fields + from InvenTree.metadata import InvenTreeMetadata metadata = InvenTreeMetadata() @@ -299,6 +304,7 @@ def available_fields(self): else: fields = {} + self._available_fields = fields return fields def required_fields(self): @@ -388,10 +394,23 @@ def clean(self): column = models.CharField(blank=True, max_length=100, verbose_name=_('Column')) + @property + def available_fields(self): + """Return a list of available fields for this import session. + + These fields get cached, as they are expensive to compute. + """ + if fields := getattr(self, '_available_fields', None): + return fields + + self._available_fields = self.session.available_fields() + + return self._available_fields + @property def field_definition(self): """Return the field definition associated with this column mapping.""" - fields = self.session.available_fields() + fields = self.available_fields return fields.get(self.field, None) @property From 7801ed7d55fb7806a1a29baedb43a1b91e1c209e Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Jul 2024 09:06:16 +0000 Subject: [PATCH 158/190] Fix calculation of completed_row_count --- src/backend/InvenTree/importer/models.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 9fe5880825c1..7e726424462e 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -278,10 +278,7 @@ def row_count(self): @property def completed_row_count(self): """Return the number of completed rows for this session.""" - if self.row_count == 0: - return 0 - - return self.rows.filter(complete=True).count() / self.row_count * 100 + return self.rows.filter(complete=True).count() def available_fields(self): """Returns information on the available fields. From 216aac31f0793b6c3a043d818d8b06995e4e8236 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Jul 2024 12:53:11 +0000 Subject: [PATCH 159/190] Import individual rows from import session --- src/backend/InvenTree/importer/api.py | 27 ++++++++++ src/backend/InvenTree/importer/models.py | 1 + src/backend/InvenTree/importer/serializers.py | 51 +++++++++++++++++++ .../importer/ImportDataSelector.tsx | 35 +++++++++++-- src/frontend/src/enums/ApiEndpoints.tsx | 1 + 5 files changed, 110 insertions(+), 5 deletions(-) diff --git a/src/backend/InvenTree/importer/api.py b/src/backend/InvenTree/importer/api.py index 7472e30025c5..b039c61063b2 100644 --- a/src/backend/InvenTree/importer/api.py +++ b/src/backend/InvenTree/importer/api.py @@ -14,6 +14,7 @@ from InvenTree.api import BulkDeleteMixin from InvenTree.filters import SEARCH_ORDER_FILTER from InvenTree.mixins import ( + CreateAPI, ListAPI, ListCreateAPI, RetrieveUpdateAPI, @@ -81,6 +82,27 @@ def post(self, request, pk): return Response(importer.serializers.DataImportSessionSerializer(session).data) +class DataImportSessionAcceptRows(CreateAPI): + """API endpoint to accept the rows for a DataImportSession.""" + + queryset = importer.models.DataImportSession.objects.all() + serializer_class = importer.serializers.DataImportAcceptRowSerializer + + def get_serializer_context(self): + """Add the import session object to the serializer context.""" + ctx = super().get_serializer_context() + + try: + ctx['session'] = importer.models.DataImportSession.objects.get( + pk=self.kwargs.get('pk', None) + ) + except Exception: + pass + + ctx['request'] = self.request + return ctx + + class DataImportColumnMappingList(ListAPI): """API endpoint for accessing a list of DataImportColumnMap objects.""" @@ -134,6 +156,11 @@ class DataImportRowDetail(RetrieveUpdateDestroyAPI): DataImportSessionAcceptFields.as_view(), name='api-import-session-accept-fields', ), + path( + 'accept_rows/', + DataImportSessionAcceptRows.as_view(), + name='api-import-session-accept-rows', + ), path( '', DataImportSessionDetail.as_view(), diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 7e726424462e..e0e16db5303a 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -555,6 +555,7 @@ def validate(self, commit=False) -> bool: serializer.save() self.complete = True self.save() + except Exception as e: self.errors = {'non_field_errors': str(e)} result = False diff --git a/src/backend/InvenTree/importer/serializers.py b/src/backend/InvenTree/importer/serializers.py index 222982c9d0ce..25ab1ab09217 100644 --- a/src/backend/InvenTree/importer/serializers.py +++ b/src/backend/InvenTree/importer/serializers.py @@ -1,5 +1,6 @@ """API serializers for the importer app.""" +from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ from rest_framework import serializers @@ -118,3 +119,53 @@ class Meta: 'valid', 'complete', ] + + +class DataImportAcceptRowSerializer(serializers.Serializer): + """Serializer for accepting rows of data.""" + + class Meta: + """Serializer meta options.""" + + fields = ['rows'] + + rows = serializers.PrimaryKeyRelatedField( + queryset=importer.models.DataImportRow.objects.all(), + many=True, + required=True, + label=_('Rows'), + help_text=_('List of row IDs to accept'), + ) + + def validate_rows(self, rows): + """Ensure that the provided rows are valid. + + - Row must point to the same import session + - Row must contain valid data + - Row must not have already been completed + """ + session = self.context.get('session', None) + + if not rows or len(rows) == 0: + raise ValidationError(_('No rows provided')) + + for row in rows: + if row.session != session: + raise ValidationError(_('Row does not belong to this session')) + + if not row.valid: + raise ValidationError(_('Row contains invalid data')) + + if row.complete: + raise ValidationError(_('Row has already been completed')) + + return rows + + def save(self): + """Complete the provided rows.""" + rows = self.validated_data['rows'] + + for row in rows: + row.validate(commit=True) + + return rows diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index aa798ba2f2b1..58367ded7119 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -1,12 +1,15 @@ import { t } from '@lingui/macro'; import { Group, HoverCard, Stack, Text } from '@mantine/core'; import { + IconCircle, IconCircleCheck, + IconCircleDashedCheck, IconExclamationCircle, IconSquareArrowRight } from '@tabler/icons-react'; import { ReactNode, useCallback, useMemo, useState } from 'react'; +import { api } from '../../App'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { cancelEvent } from '../../functions/events'; import { @@ -128,6 +131,23 @@ export default function ImporterDataSelector({ return fields; }, [selectedFieldNames, session.availableFields]); + const importData = useCallback( + (rows: number[]) => { + api + .post( + apiUrl(ApiEndpoints.import_session_accept_rows, session.sessionId), + { + rows: rows + } + ) + .catch(() => {}) + .finally(() => { + table.refreshTable(); + }); + }, + [session.sessionId, table.refreshTable] + ); + const [selectedRow, setSelectedRow] = useState({}); const editCell = useCallback( @@ -191,9 +211,11 @@ export default function ImporterDataSelector({ return ( {row.row_index} - {row.valid ? ( - - ) : ( + {row.complete && } + {!row.complete && row.valid && ( + + )} + {!row.complete && !row.valid && ( @@ -244,7 +266,10 @@ export default function ImporterDataSelector({ title: t`Accept`, icon: , color: 'green', - hidden: record.complete + hidden: record.complete, + onClick: () => { + importData([record.pk]); + } }, RowEditAction({ hidden: record.complete, @@ -264,7 +289,7 @@ export default function ImporterDataSelector({ }) ]; }, - [session] + [session, importData] ); const filters: TableFilter[] = useMemo(() => { diff --git a/src/frontend/src/enums/ApiEndpoints.tsx b/src/frontend/src/enums/ApiEndpoints.tsx index ab4ad7c83ca9..e7d9e1747fbf 100644 --- a/src/frontend/src/enums/ApiEndpoints.tsx +++ b/src/frontend/src/enums/ApiEndpoints.tsx @@ -49,6 +49,7 @@ export enum ApiEndpoints { // Data import endpoints import_session_list = 'importer/session/', import_session_accept_fields = 'importer/session/:id/accept_fields/', + import_session_accept_rows = 'importer/session/:id/accept_rows/', import_session_column_mapping_list = 'importer/column-mapping/', import_session_row_list = 'importer/row/', From a80886972b34d940badf0a876e6706786ccfba57 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Jul 2024 13:02:09 +0000 Subject: [PATCH 160/190] Allow import of multiple simultaneous records --- .../importer/ImportDataSelector.tsx | 22 ++++++++++++++++++- .../tables/settings/ImportSessionTable.tsx | 1 + 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 58367ded7119..47dcfea2356a 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -1,7 +1,6 @@ import { t } from '@lingui/macro'; import { Group, HoverCard, Stack, Text } from '@mantine/core'; import { - IconCircle, IconCircleCheck, IconCircleDashedCheck, IconExclamationCircle, @@ -23,6 +22,7 @@ import { TableColumn } from '../../tables/Column'; import { TableFilter } from '../../tables/Filter'; import { InvenTreeTable } from '../../tables/InvenTreeTable'; import { RowDeleteAction, RowEditAction } from '../../tables/RowActions'; +import { ActionButton } from '../buttons/ActionButton'; import { YesNoButton } from '../buttons/YesNoButton'; import { ApiFormFieldSet } from '../forms/fields/ApiFormField'; @@ -309,6 +309,25 @@ export default function ImporterDataSelector({ ]; }, []); + const tableActions = useMemo(() => { + // Can only "import" valid (and incomplete) rows + const canImport: boolean = + table.hasSelectedRecords && + table.selectedRecords.every((row: any) => row.valid && !row.complete); + + return [ + } + color="green" + tooltip={t`Import selected rows`} + onClick={() => { + importData(table.selectedRecords.map((row: any) => row.pk)); + }} + /> + ]; + }, [table.hasSelectedRecords, table.selectedRecords]); + return ( <> {editRow.modal} @@ -323,6 +342,7 @@ export default function ImporterDataSelector({ session: session.sessionId }, rowActions: rowActions, + tableActions: tableActions, tableFilters: filters, enableColumnSwitching: true, enableColumnCaching: false, diff --git a/src/frontend/src/tables/settings/ImportSessionTable.tsx b/src/frontend/src/tables/settings/ImportSessionTable.tsx index 47bbd364bdab..6c71f5251ac2 100644 --- a/src/frontend/src/tables/settings/ImportSessionTable.tsx +++ b/src/frontend/src/tables/settings/ImportSessionTable.tsx @@ -172,6 +172,7 @@ export default function ImportSesssionTable() { onClose={() => { setSelectedSession(undefined); setOpened(false); + table.refreshTable(); }} /> From d8e4a1d0f424d6b1ffb19b0ddb494ba6b6fa76b0 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Jul 2024 14:44:26 +0000 Subject: [PATCH 161/190] Improve extraction of metadata - Especially for related fields - Request object no longer required --- src/backend/InvenTree/InvenTree/metadata.py | 8 ++++---- src/backend/InvenTree/build/models.py | 2 +- src/backend/InvenTree/importer/models.py | 12 +++++++++--- src/backend/InvenTree/order/models.py | 6 +++--- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/metadata.py b/src/backend/InvenTree/InvenTree/metadata.py index c810bbdbe4cf..b82ffa849aa1 100644 --- a/src/backend/InvenTree/InvenTree/metadata.py +++ b/src/backend/InvenTree/InvenTree/metadata.py @@ -160,6 +160,8 @@ def get_serializer_info(self, serializer): """Override get_serializer_info so that we can add 'default' values to any fields whose Meta.model specifies a default value.""" self.serializer = serializer + request = getattr(self, 'request', None) + serializer_info = super().get_serializer_info(serializer) # Look for any dynamic fields which were not available when the serializer was instantiated @@ -187,10 +189,8 @@ def get_serializer_info(self, serializer): model_fields = model_meta.get_field_info(model_class) - model_default_func = getattr(model_class, 'api_defaults', None) - - if model_default_func: - model_default_values = model_class.api_defaults(self.request) + if model_default_func := getattr(model_class, 'api_defaults', None): + model_default_values = model_default_func(request=request) or {} else: model_default_values = {} diff --git a/src/backend/InvenTree/build/models.py b/src/backend/InvenTree/build/models.py index 7430eb4a29b2..f7e2d2eda655 100644 --- a/src/backend/InvenTree/build/models.py +++ b/src/backend/InvenTree/build/models.py @@ -104,7 +104,7 @@ def api_instance_filters(self): } @classmethod - def api_defaults(cls, request): + def api_defaults(cls, request=None): """Return default values for this model when issuing an API OPTIONS request.""" defaults = { 'reference': generate_next_build_reference(), diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index e0e16db5303a..c7b7b1eea8f7 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -1,7 +1,6 @@ """Model definitions for the 'importer' app.""" import logging -from typing import Collection from django.contrib.auth.models import User from django.core.exceptions import ValidationError as DjangoValidationError @@ -414,13 +413,20 @@ def field_definition(self): def label(self): """Extract the 'label' associated with the mapped field.""" if field_def := self.field_definition: - return field_def.label + return field_def.get('label', None) @property def description(self): """Extract the 'description' associated with the mapped field.""" + description = None + if field_def := self.field_definition: - return field_def.help_text + description = field_def.get('help_text', None) + + if not description: + description = self.label + + return description class DataImportRow(models.Model): diff --git a/src/backend/InvenTree/order/models.py b/src/backend/InvenTree/order/models.py index f4abb444cd54..1a29a9941b8f 100644 --- a/src/backend/InvenTree/order/models.py +++ b/src/backend/InvenTree/order/models.py @@ -400,7 +400,7 @@ def get_status_class(cls): return PurchaseOrderStatusGroups @classmethod - def api_defaults(cls, request): + def api_defaults(cls, request=None): """Return default values for this model when issuing an API OPTIONS request.""" defaults = { 'reference': order.validators.generate_next_purchase_order_reference() @@ -865,7 +865,7 @@ def get_status_class(cls): return SalesOrderStatusGroups @classmethod - def api_defaults(cls, request): + def api_defaults(cls, request=None): """Return default values for this model when issuing an API OPTIONS request.""" defaults = {'reference': order.validators.generate_next_sales_order_reference()} @@ -2027,7 +2027,7 @@ def get_status_class(cls): return ReturnOrderStatusGroups @classmethod - def api_defaults(cls, request): + def api_defaults(cls, request=None): """Return default values for this model when issuing an API OPTIONS request.""" defaults = { 'reference': order.validators.generate_next_return_order_reference() From 363a8ab5e320315adac1c8aa7734e81a09d74584 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Jul 2024 14:44:52 +0000 Subject: [PATCH 162/190] Implement suspended rendering of model instances --- .../importer/ImportDataSelector.tsx | 14 ++++++++++ .../src/components/render/Instance.tsx | 26 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 47dcfea2356a..57a1a125488c 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -6,10 +6,12 @@ import { IconExclamationCircle, IconSquareArrowRight } from '@tabler/icons-react'; +import { useSuspenseQuery } from '@tanstack/react-query'; import { ReactNode, useCallback, useMemo, useState } from 'react'; import { api } from '../../App'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; +import { ModelType } from '../../enums/ModelType'; import { cancelEvent } from '../../functions/events'; import { useDeleteApiFormModal, @@ -25,6 +27,8 @@ import { RowDeleteAction, RowEditAction } from '../../tables/RowActions'; import { ActionButton } from '../buttons/ActionButton'; import { YesNoButton } from '../buttons/YesNoButton'; import { ApiFormFieldSet } from '../forms/fields/ApiFormField'; +import { RenderInstance, RenderSuspendedInstance } from '../render/Instance'; +import { ModelInformationDict } from '../render/ModelType'; function ImporterDataCell({ session, @@ -62,6 +66,16 @@ function ImporterDataCell({ return ( ); + case 'related field': + if (field_def.model && row.data[column.field]) { + return ( + + ); + } + break; default: break; } diff --git a/src/frontend/src/components/render/Instance.tsx b/src/frontend/src/components/render/Instance.tsx index 4424045d9cab..c3f461640c64 100644 --- a/src/frontend/src/components/render/Instance.tsx +++ b/src/frontend/src/components/render/Instance.tsx @@ -1,9 +1,12 @@ import { t } from '@lingui/macro'; import { Alert, Anchor, Group, Space, Text } from '@mantine/core'; +import { useSuspenseQuery } from '@tanstack/react-query'; import { ReactNode, useCallback } from 'react'; +import { api } from '../../App'; import { ModelType } from '../../enums/ModelType'; import { navigateToLink } from '../../functions/navigation'; +import { apiUrl } from '../../states/ApiState'; import { Thumbnail } from '../images/Thumbnail'; import { RenderBuildLine, RenderBuildOrder } from './Build'; import { @@ -14,6 +17,7 @@ import { RenderSupplierPart } from './Company'; import { RenderImportSession, RenderProjectCode } from './Generic'; +import { ModelInformationDict } from './ModelType'; import { RenderPurchaseOrder, RenderReturnOrder, @@ -104,6 +108,28 @@ export function RenderInstance(props: RenderInstanceProps): ReactNode { return ; } +export function RenderSuspendedInstance({ + model, + pk +}: { + model: ModelType; + pk: number; +}): ReactNode { + const { data } = useSuspenseQuery({ + queryKey: ['model', model, pk], + queryFn: async () => { + const url = apiUrl(ModelInformationDict[model].api_endpoint, pk); + + return api + .get(url) + .then((response) => response.data) + .catch(() => null); + } + }); + + return ; +} + /** * Helper function for rendering an inline model in a consistent style */ From 12c9c0163a562f658afeba1d377743fe4c862d3e Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Jul 2024 15:04:58 +0000 Subject: [PATCH 163/190] Cleanup --- .../importer/ImportDataSelector.tsx | 32 ++++++++-------- .../components/importer/ImporterDrawer.tsx | 37 ++++++------------- 2 files changed, 27 insertions(+), 42 deletions(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 57a1a125488c..e3b4ef495077 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -6,12 +6,10 @@ import { IconExclamationCircle, IconSquareArrowRight } from '@tabler/icons-react'; -import { useSuspenseQuery } from '@tanstack/react-query'; import { ReactNode, useCallback, useMemo, useState } from 'react'; import { api } from '../../App'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; -import { ModelType } from '../../enums/ModelType'; import { cancelEvent } from '../../functions/events'; import { useDeleteApiFormModal, @@ -27,8 +25,7 @@ import { RowDeleteAction, RowEditAction } from '../../tables/RowActions'; import { ActionButton } from '../buttons/ActionButton'; import { YesNoButton } from '../buttons/YesNoButton'; import { ApiFormFieldSet } from '../forms/fields/ApiFormField'; -import { RenderInstance, RenderSuspendedInstance } from '../render/Instance'; -import { ModelInformationDict } from '../render/ModelType'; +import { RenderSuspendedInstance } from '../render/Instance'; function ImporterDataCell({ session, @@ -44,9 +41,12 @@ function ImporterDataCell({ const onRowEdit = useCallback( (event: any) => { cancelEvent(event); - onEdit?.(); + + if (!row.complete) { + onEdit?.(); + } }, - [onEdit] + [onEdit, row] ); const cellErrors: string[] = useMemo(() => { @@ -57,8 +57,6 @@ function ImporterDataCell({ }, [row.errors, column.field]); const cellValue: ReactNode = useMemo(() => { - // TODO: Render inline models, rather than raw PK values - let field_def = session.availableFields[column.field]; switch (field_def?.type) { @@ -164,15 +162,6 @@ export default function ImporterDataSelector({ const [selectedRow, setSelectedRow] = useState({}); - const editCell = useCallback( - (row: any, col: any) => { - setSelectedRow(row); - setSelectedFieldNames([col.field]); - editRow.open(); - }, - [session] - ); - const editRow = useEditApiFormModal({ url: ApiEndpoints.import_session_row_list, pk: selectedRow.pk, @@ -191,6 +180,15 @@ export default function ImporterDataSelector({ onFormSuccess: (row: any) => table.updateRecord(row) }); + const editCell = useCallback( + (row: any, col: any) => { + setSelectedRow(row); + setSelectedFieldNames([col.field]); + editRow.open(); + }, + [session, editRow] + ); + const deleteRow = useDeleteApiFormModal({ url: ApiEndpoints.import_session_row_list, pk: selectedRow.pk, diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index cee5c2c08667..652a44783f35 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -14,11 +14,14 @@ import { import { IconCircleX } from '@tabler/icons-react'; import { ReactNode, useCallback, useMemo, useState } from 'react'; +import { ModelType } from '../../enums/ModelType'; import { ImportSessionStatus, useImportSession } from '../../hooks/UseImportSession'; +import { ProgressBar } from '../items/ProgressBar'; import { StylishText } from '../items/StylishText'; +import { StatusRenderer } from '../render/StatusRenderer'; import ImporterDataSelector from './ImportDataSelector'; import ImporterColumnSelector from './ImporterColumnSelector'; @@ -57,25 +60,6 @@ export default function ImporterDrawer({ }) { const session = useImportSession({ sessionId: sessionId }); - const [currentStep, setCurrentStep] = useState(1); - - const description: string = useMemo(() => { - switch (session.status) { - case ImportSessionStatus.INITIAL: - return t`Data File Upload`; - case ImportSessionStatus.MAPPING: - return t`Mapping Data Columns`; - case ImportSessionStatus.IMPORTING: - return t`Importing Data`; - case ImportSessionStatus.PROCESSING: - return t`Processing Data`; - case ImportSessionStatus.COMPLETE: - return t`Import Complete`; - default: - return t`Unknown Status` + ` - ${session.status}`; - } - }, [session]); - const widget = useMemo(() => { switch (session.status) { case ImportSessionStatus.INITIAL: @@ -99,16 +83,19 @@ export default function ImporterDrawer({ {session.sessionData?.statusText ?? t`Importing Data`} - - - + {StatusRenderer({ + status: session.status, + type: ModelType.importsession + })} + + @@ -116,7 +103,7 @@ export default function ImporterDrawer({ ); - }, []); + }, [session.sessionData]); return ( Date: Wed, 3 Jul 2024 23:22:08 +0000 Subject: [PATCH 164/190] Implement more columns for StockTable --- .../src/tables/stock/StockItemTable.tsx | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/frontend/src/tables/stock/StockItemTable.tsx b/src/frontend/src/tables/stock/StockItemTable.tsx index 60df40ad9be9..2b21ce13c15c 100644 --- a/src/frontend/src/tables/stock/StockItemTable.tsx +++ b/src/frontend/src/tables/stock/StockItemTable.tsx @@ -4,7 +4,7 @@ import { ReactNode, useMemo } from 'react'; import { AddItemButton } from '../../components/buttons/AddItemButton'; import { ActionDropdown } from '../../components/items/ActionDropdown'; -import { formatCurrency } from '../../defaults/formatters'; +import { formatCurrency, formatPriceRange } from '../../defaults/formatters'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { ModelType } from '../../enums/ModelType'; import { UserRoles } from '../../enums/Roles'; @@ -237,10 +237,29 @@ function stockItemTableColumns(): TableColumn[] { formatCurrency(record.purchase_price, { currency: record.purchase_price_currency }) + }, + { + accessor: 'packaging', + sortable: true + }, + { + accessor: 'stock_value', + title: t`Stock Value`, + sortable: false, + render: (record: any) => { + let min_price = + record.purchase_price || record.part_detail?.pricing_min; + let max_price = + record.purchase_price || record.part_detail?.pricing_max; + let currency = record.purchase_price_currency || undefined; + + return formatPriceRange(min_price, max_price, currency); + } + }, + { + accessor: 'notes', + sortable: false } - // TODO: stock value - // TODO: packaging - // TODO: notes ]; } From 6812601b40b9f550f71f56a41c5cd148e9119d23 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Jul 2024 23:27:49 +0000 Subject: [PATCH 165/190] Allow stock filtering by packaging field --- src/backend/InvenTree/stock/api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/InvenTree/stock/api.py b/src/backend/InvenTree/stock/api.py index 3e93d8eb3380..4ea5be31171b 100644 --- a/src/backend/InvenTree/stock/api.py +++ b/src/backend/InvenTree/stock/api.py @@ -1190,6 +1190,7 @@ def filter_queryset(self, queryset): 'updated', 'stocktake_date', 'expiry_date', + 'packaging', 'quantity', 'stock', 'status', From 6e8481434084f9c4dd753ea9aa83f3e6e1d6b6e8 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Wed, 3 Jul 2024 23:33:32 +0000 Subject: [PATCH 166/190] Fix "stock_value" column --- src/frontend/src/tables/stock/StockItemTable.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/frontend/src/tables/stock/StockItemTable.tsx b/src/frontend/src/tables/stock/StockItemTable.tsx index 2b21ce13c15c..9f6e9fcd3495 100644 --- a/src/frontend/src/tables/stock/StockItemTable.tsx +++ b/src/frontend/src/tables/stock/StockItemTable.tsx @@ -253,7 +253,10 @@ function stockItemTableColumns(): TableColumn[] { record.purchase_price || record.part_detail?.pricing_max; let currency = record.purchase_price_currency || undefined; - return formatPriceRange(min_price, max_price, currency); + return formatPriceRange(min_price, max_price, { + currency: currency, + multiplier: record.quantity + }); } }, { From 87af1fb2ea05c0534fe65649c0a93ff1fb9e3779 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 00:43:00 +0000 Subject: [PATCH 167/190] Improve metadata extraction - Handle read_only_fields in Meta - Handle write_only_fields in Meta --- src/backend/InvenTree/InvenTree/metadata.py | 20 +++++++++++++++++++ src/backend/InvenTree/importer/mixins.py | 16 +++++++++++++++ src/backend/InvenTree/stock/serializers.py | 22 +++++++++++++++------ 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/metadata.py b/src/backend/InvenTree/InvenTree/metadata.py index b82ffa849aa1..7805d73c0a77 100644 --- a/src/backend/InvenTree/InvenTree/metadata.py +++ b/src/backend/InvenTree/InvenTree/metadata.py @@ -176,6 +176,14 @@ def get_serializer_info(self, serializer): model_class = None + # Extract read_only_fields and write_only_fields from the Meta class (if available) + if meta := getattr(serializer, 'Meta', None): + read_only_fields = getattr(meta, 'read_only_fields', []) + write_only_fields = getattr(meta, 'write_only_fields', []) + else: + read_only_fields = [] + write_only_fields = [] + # Attributes to copy extra attributes from the model to the field (if they don't exist) # Note that the attributes may be named differently on the underlying model! extra_attributes = { @@ -197,6 +205,12 @@ def get_serializer_info(self, serializer): # Iterate through simple fields for name, field in model_fields.fields.items(): if name in serializer_info.keys(): + if name in read_only_fields: + serializer_info[name]['read_only'] = True + + if name in write_only_fields: + serializer_info[name]['write_only'] = True + if field.has_default(): default = field.default @@ -230,6 +244,12 @@ def get_serializer_info(self, serializer): # Ignore reverse relations continue + if name in read_only_fields: + serializer_info[name]['read_only'] = True + + if name in write_only_fields: + serializer_info[name]['write_only'] = True + # Extract and provide the "limit_choices_to" filters # This is used to automatically filter AJAX requests serializer_info[name]['filters'] = ( diff --git a/src/backend/InvenTree/importer/mixins.py b/src/backend/InvenTree/importer/mixins.py index a5a4624a39f7..e0e064afc43e 100644 --- a/src/backend/InvenTree/importer/mixins.py +++ b/src/backend/InvenTree/importer/mixins.py @@ -60,11 +60,19 @@ def get_importable_fields(self) -> dict: """ importable_fields = {} + if meta := getattr(self, 'Meta', None): + read_only_fields = getattr(meta, 'read_only_fields', []) + else: + read_only_fields = [] + for name, field in self.fields.items(): # Skip read-only fields if getattr(field, 'read_only', False): continue + if name in read_only_fields: + continue + # Skip fields which are themselves serializers if issubclass(field.__class__, serializers.Serializer): continue @@ -121,11 +129,19 @@ def get_exportable_fields(self) -> dict: """ fields = {} + if meta := getattr(self, 'Meta', None): + write_only_fields = getattr(meta, 'write_only_fields', []) + else: + write_only_fields = [] + for name, field in self.fields.items(): # Skip write-only fields if getattr(field, 'write_only', False): continue + if name in write_only_fields: + continue + # Skip fields which are themselves serializers if issubclass(field.__class__, serializers.Serializer): continue diff --git a/src/backend/InvenTree/stock/serializers.py b/src/backend/InvenTree/stock/serializers.py index a060bbc6b6cf..fd3a889847b8 100644 --- a/src/backend/InvenTree/stock/serializers.py +++ b/src/backend/InvenTree/stock/serializers.py @@ -560,12 +560,22 @@ def annotate_queryset(queryset): quantity = InvenTreeDecimalField() # Annotated fields - allocated = serializers.FloatField(required=False) - expired = serializers.BooleanField(required=False, read_only=True) - installed_items = serializers.IntegerField(read_only=True, required=False) - child_items = serializers.IntegerField(read_only=True, required=False) - stale = serializers.BooleanField(required=False, read_only=True) - tracking_items = serializers.IntegerField(read_only=True, required=False) + allocated = serializers.FloatField( + required=False, read_only=True, label=_('Allocated Quantity') + ) + expired = serializers.BooleanField( + required=False, read_only=True, label=_('Expired') + ) + installed_items = serializers.IntegerField( + read_only=True, required=False, label=_('Installed Items') + ) + child_items = serializers.IntegerField( + read_only=True, required=False, label=_('Child Items') + ) + stale = serializers.BooleanField(required=False, read_only=True, label=_('Stale')) + tracking_items = serializers.IntegerField( + read_only=True, required=False, label=_('Tracking Items') + ) purchase_price = InvenTree.serializers.InvenTreeMoneySerializer( label=_('Purchase Price'), From cfa169f33f1f0b0112b6c79d6496a4702751741f Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 00:43:16 +0000 Subject: [PATCH 168/190] Increase maximum number of importable rows --- src/backend/InvenTree/importer/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/InvenTree/importer/validators.py b/src/backend/InvenTree/importer/validators.py index 74f056d306a7..34e48b1862c3 100644 --- a/src/backend/InvenTree/importer/validators.py +++ b/src/backend/InvenTree/importer/validators.py @@ -7,7 +7,7 @@ # Define maximum limits for imported file data IMPORTER_MAX_FILE_SIZE = 32 * 1024 * 1042 -IMPORTER_MAX_ROWS = 1000 +IMPORTER_MAX_ROWS = 5000 IMPORTER_MAX_COLS = 1000 From f82c74a08bf469cb8b5b3197835b68f695cf1326 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 00:43:44 +0000 Subject: [PATCH 169/190] Force data import to run on background worker --- src/backend/InvenTree/importer/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index c7b7b1eea8f7..49b2237a084b 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -229,7 +229,7 @@ def trigger_data_import(self): self.status = DataImportStatusCode.IMPORTING.value self.save() - offload_task(importer.tasks.import_data, self.pk) + offload_task(importer.tasks.import_data, self.pk, force_async=True) def import_data(self): """Perform the data import process for this session.""" From 325ab90712d361b07866804b37625c97d8f60400 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 03:20:04 +0000 Subject: [PATCH 170/190] Add export-only fields to StockItemSerializer class --- src/backend/InvenTree/stock/serializers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/backend/InvenTree/stock/serializers.py b/src/backend/InvenTree/stock/serializers.py index fd3a889847b8..26f393f0b93f 100644 --- a/src/backend/InvenTree/stock/serializers.py +++ b/src/backend/InvenTree/stock/serializers.py @@ -333,6 +333,8 @@ class StockItemSerializer( export_exclude_fields = ['tracking_items'] + export_only_fields = ['part_pricing_min', 'part_pricing_max'] + import_exclude_fields = ['use_pack_size', 'tags'] class Meta: @@ -384,6 +386,9 @@ class Meta: 'stale', 'tracking_items', 'tags', + # Export only fields + 'part_pricing_min', + 'part_pricing_max', ] """ @@ -596,6 +601,18 @@ def annotate_queryset(queryset): tags = TagListSerializerField(required=False) + part_pricing_min = InvenTree.serializers.InvenTreeMoneySerializer( + source='part.pricing_data.overall_min', + read_only=True, + label=_('Minimum Pricing'), + ) + + part_pricing_max = InvenTree.serializers.InvenTreeMoneySerializer( + source='part.pricing_data.overall_max', + read_only=True, + label=_('Maximum Pricing'), + ) + class SerializeStockItemSerializer(serializers.Serializer): """A DRF serializer for "serializing" a StockItem. From 53469368e672d833ab1e5bfc79a69e2d64ee9a49 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 04:54:02 +0000 Subject: [PATCH 171/190] Data conversion when performing initial import --- src/backend/InvenTree/importer/models.py | 12 ++++++++++++ src/frontend/src/components/render/Instance.tsx | 4 ++++ 2 files changed, 16 insertions(+) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 49b2237a084b..f794a90b34f3 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -476,6 +476,8 @@ def extract_data(self, field_mapping: dict = None): if not field_mapping: field_mapping = self.session.field_mapping + available_fields = self.session.available_fields() + override_values = self.session.field_overrides or {} default_values = self.session.field_defaults or {} @@ -492,8 +494,18 @@ def extract_data(self, field_mapping: dict = None): if not col: continue + # Extract field type + field_def = available_fields.get(field, {}) + + field_type = field_def.get('type', None) + value = self.row_data.get(col, None) + if field_type == 'boolean': + value = InvenTree.helpers.str2bool(value) + elif field_type == 'date': + value = value or None + # Use the default value, if provided if value in [None, ''] and field in default_values: value = default_values[field] diff --git a/src/frontend/src/components/render/Instance.tsx b/src/frontend/src/components/render/Instance.tsx index c3f461640c64..9d3a4b5c9414 100644 --- a/src/frontend/src/components/render/Instance.tsx +++ b/src/frontend/src/components/render/Instance.tsx @@ -127,6 +127,10 @@ export function RenderSuspendedInstance({ } }); + if (!data) { + return ${pk}; + } + return ; } From debf6883c5a8e8d72068f268a5ae7278ca106c28 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 09:15:34 +0000 Subject: [PATCH 172/190] Various tweaks --- src/backend/InvenTree/part/serializers.py | 2 +- .../importer/ImportDataSelector.tsx | 31 ++++++++++--- .../components/importer/ImporterDrawer.tsx | 4 +- .../importer/ImporterImportProgress.tsx | 46 +++++++++++++++++++ .../src/components/render/Instance.tsx | 10 ++-- 5 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 src/frontend/src/components/importer/ImporterImportProgress.tsx diff --git a/src/backend/InvenTree/part/serializers.py b/src/backend/InvenTree/part/serializers.py index 485db6efbd7a..9a7ece80f82d 100644 --- a/src/backend/InvenTree/part/serializers.py +++ b/src/backend/InvenTree/part/serializers.py @@ -702,7 +702,7 @@ def __init__(self, *args, **kwargs): # Fields required for certain operations, but are not part of the model if f in ['remote_image', 'existing_image']: continue - self.fields.pop(f) + self.fields.pop(f, None) if not pricing: self.fields.pop('pricing_min', None) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index e3b4ef495077..92c972005066 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -1,10 +1,11 @@ import { t } from '@lingui/macro'; import { Group, HoverCard, Stack, Text } from '@mantine/core'; +import { notifications } from '@mantine/notifications'; import { + IconArrowRight, IconCircleCheck, IconCircleDashedCheck, - IconExclamationCircle, - IconSquareArrowRight + IconExclamationCircle } from '@tabler/icons-react'; import { ReactNode, useCallback, useMemo, useState } from 'react'; @@ -145,6 +146,15 @@ export default function ImporterDataSelector({ const importData = useCallback( (rows: number[]) => { + notifications.show({ + title: t`Importing Rows`, + message: t`Please wait while the data is imported`, + autoClose: false, + color: 'blue', + id: 'importing-rows', + icon: + }); + api .post( apiUrl(ApiEndpoints.import_session_accept_rows, session.sessionId), @@ -152,8 +162,17 @@ export default function ImporterDataSelector({ rows: rows } ) - .catch(() => {}) + .catch(() => { + notifications.show({ + title: t`Error`, + message: t`An error occurred while importing data`, + color: 'red', + autoClose: true + }); + }) .finally(() => { + table.clearSelectedRecords(); + notifications.hide('importing-rows'); table.refreshTable(); }); }, @@ -276,9 +295,9 @@ export default function ImporterDataSelector({ return [ { title: t`Accept`, - icon: , + icon: , color: 'green', - hidden: record.complete, + hidden: record.complete || !record.valid, onClick: () => { importData([record.pk]); } @@ -330,7 +349,7 @@ export default function ImporterDataSelector({ return [ } + icon={} color="green" tooltip={t`Import selected rows`} onClick={() => { diff --git a/src/frontend/src/components/importer/ImporterDrawer.tsx b/src/frontend/src/components/importer/ImporterDrawer.tsx index 652a44783f35..36507d4627b3 100644 --- a/src/frontend/src/components/importer/ImporterDrawer.tsx +++ b/src/frontend/src/components/importer/ImporterDrawer.tsx @@ -19,11 +19,11 @@ import { ImportSessionStatus, useImportSession } from '../../hooks/UseImportSession'; -import { ProgressBar } from '../items/ProgressBar'; import { StylishText } from '../items/StylishText'; import { StatusRenderer } from '../render/StatusRenderer'; import ImporterDataSelector from './ImportDataSelector'; import ImporterColumnSelector from './ImporterColumnSelector'; +import ImporterImportProgress from './ImporterImportProgress'; /* * Stepper component showing the current step of the data import process. @@ -67,7 +67,7 @@ export default function ImporterDrawer({ case ImportSessionStatus.MAPPING: return ; case ImportSessionStatus.IMPORTING: - return Importing...; + return ; case ImportSessionStatus.PROCESSING: return ; case ImportSessionStatus.COMPLETE: diff --git a/src/frontend/src/components/importer/ImporterImportProgress.tsx b/src/frontend/src/components/importer/ImporterImportProgress.tsx new file mode 100644 index 000000000000..e44bcee76e02 --- /dev/null +++ b/src/frontend/src/components/importer/ImporterImportProgress.tsx @@ -0,0 +1,46 @@ +import { t } from '@lingui/macro'; +import { Center, Container, Loader, Stack, Text } from '@mantine/core'; +import { useInterval } from '@mantine/hooks'; +import { useEffect } from 'react'; + +import { + ImportSessionState, + ImportSessionStatus +} from '../../hooks/UseImportSession'; +import { StylishText } from '../items/StylishText'; + +export default function ImporterImportProgress({ + session +}: { + session: ImportSessionState; +}) { + // Periodically refresh the import session data + const interval = useInterval(() => { + console.log('refreshing:', session.status); + + if (session.status == ImportSessionStatus.IMPORTING) { + session.refreshSession(); + } + }, 1000); + + useEffect(() => { + interval.start(); + return interval.stop; + }, []); + + return ( + <> +
+ + + {t`Importing Records`} + + + {t`Imported rows`}: {session.sessionData.row_count} + + + +
+ + ); +} diff --git a/src/frontend/src/components/render/Instance.tsx b/src/frontend/src/components/render/Instance.tsx index 9d3a4b5c9414..b977b8688ef6 100644 --- a/src/frontend/src/components/render/Instance.tsx +++ b/src/frontend/src/components/render/Instance.tsx @@ -1,6 +1,6 @@ import { t } from '@lingui/macro'; -import { Alert, Anchor, Group, Space, Text } from '@mantine/core'; -import { useSuspenseQuery } from '@tanstack/react-query'; +import { Alert, Anchor, Group, Skeleton, Space, Text } from '@mantine/core'; +import { useQuery, useSuspenseQuery } from '@tanstack/react-query'; import { ReactNode, useCallback } from 'react'; import { api } from '../../App'; @@ -115,7 +115,7 @@ export function RenderSuspendedInstance({ model: ModelType; pk: number; }): ReactNode { - const { data } = useSuspenseQuery({ + const { data, isLoading, isFetching } = useQuery({ queryKey: ['model', model, pk], queryFn: async () => { const url = apiUrl(ModelInformationDict[model].api_endpoint, pk); @@ -127,6 +127,10 @@ export function RenderSuspendedInstance({ } }); + if (isLoading || isFetching) { + return ; + } + if (!data) { return ${pk}; } From c6f6f5f8ed8da9821fee7b63696eddf81db94a2f Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 09:22:44 +0000 Subject: [PATCH 173/190] Fix order of operations for data import --- src/backend/InvenTree/importer/models.py | 10 +++++----- .../src/components/importer/ImportDataSelector.tsx | 4 ++++ src/frontend/src/tables/InvenTreeTable.tsx | 3 +++ 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index f794a90b34f3..8f3528c920d4 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -257,18 +257,18 @@ def import_data(self): ) ) - # Finally, create the DataImportRow objects + # Create the DataImportRow objects importer.models.DataImportRow.objects.bulk_create(row_objects) - # Mark the import task as "PROCESSING" - self.status = DataImportStatusCode.PROCESSING.value - self.save() - # Set initial data and errors for each row for row in self.rows.all(): row.extract_data(field_mapping=self.field_mapping) row.validate() + # Mark the import task as "PROCESSING" + self.status = DataImportStatusCode.PROCESSING.value + self.save() + @property def row_count(self): """Return the number of rows in the import session.""" diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 92c972005066..4d62be2dc8d4 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -60,6 +60,10 @@ function ImporterDataCell({ const cellValue: ReactNode = useMemo(() => { let field_def = session.availableFields[column.field]; + if (!row?.data) { + return '-'; + } + switch (field_def?.type) { case 'boolean': return ( diff --git a/src/frontend/src/tables/InvenTreeTable.tsx b/src/frontend/src/tables/InvenTreeTable.tsx index 2bad26176f8f..cbc3eda0386d 100644 --- a/src/frontend/src/tables/InvenTreeTable.tsx +++ b/src/frontend/src/tables/InvenTreeTable.tsx @@ -551,6 +551,9 @@ export function InvenTreeTable({ message: t`Failed to delete records`, color: 'red' }); + }) + .finally(() => { + tableState.clearSelectedRecords(); }); } }); From 133b31b88bb0834a2110b7a17f2d3ff1a0ea52ab Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 09:25:29 +0000 Subject: [PATCH 174/190] Rename component --- src/frontend/src/components/importer/ImportDataSelector.tsx | 4 ++-- src/frontend/src/components/render/Instance.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index 4d62be2dc8d4..e546398fecbc 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -26,7 +26,7 @@ import { RowDeleteAction, RowEditAction } from '../../tables/RowActions'; import { ActionButton } from '../buttons/ActionButton'; import { YesNoButton } from '../buttons/YesNoButton'; import { ApiFormFieldSet } from '../forms/fields/ApiFormField'; -import { RenderSuspendedInstance } from '../render/Instance'; +import { RenderRemoteInstance } from '../render/Instance'; function ImporterDataCell({ session, @@ -72,7 +72,7 @@ function ImporterDataCell({ case 'related field': if (field_def.model && row.data[column.field]) { return ( - diff --git a/src/frontend/src/components/render/Instance.tsx b/src/frontend/src/components/render/Instance.tsx index b977b8688ef6..cc179631c24c 100644 --- a/src/frontend/src/components/render/Instance.tsx +++ b/src/frontend/src/components/render/Instance.tsx @@ -108,7 +108,7 @@ export function RenderInstance(props: RenderInstanceProps): ReactNode { return ; } -export function RenderSuspendedInstance({ +export function RenderRemoteInstance({ model, pk }: { From fa50c0efaf2084207adb901ee289bed2f2538644 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 09:41:20 +0000 Subject: [PATCH 175/190] Allow import/export of more model types --- src/backend/InvenTree/common/api.py | 5 +++-- .../common/migrations/0018_projectcode.py | 3 +++ .../common/migrations/0020_customunit.py | 3 +++ src/backend/InvenTree/common/models.py | 10 ++++++++++ src/backend/InvenTree/common/serializers.py | 8 ++++++-- src/backend/InvenTree/part/api.py | 16 +++++++++------- src/backend/InvenTree/part/serializers.py | 8 ++++++-- .../components/importer/ImportDataSelector.tsx | 12 +++++++++--- src/frontend/src/tables/InvenTreeTable.tsx | 12 ++++++------ .../src/tables/part/PartParameterTable.tsx | 1 + .../tables/part/PartParameterTemplateTable.tsx | 3 ++- .../src/tables/settings/CustomUnitsTable.tsx | 3 ++- .../src/tables/settings/ProjectCodeTable.tsx | 3 ++- 13 files changed, 62 insertions(+), 25 deletions(-) diff --git a/src/backend/InvenTree/common/api.py b/src/backend/InvenTree/common/api.py index 9819543a5be8..a1063b5d09d6 100644 --- a/src/backend/InvenTree/common/api.py +++ b/src/backend/InvenTree/common/api.py @@ -27,6 +27,7 @@ import common.serializers from common.settings import get_global_setting from generic.states.api import AllStatusViews, StatusView +from importer.mixins import DataExportViewMixin from InvenTree.api import BulkDeleteMixin, MetadataView from InvenTree.config import CONFIG_LOOKUPS from InvenTree.filters import ORDER_FILTER, SEARCH_ORDER_FILTER @@ -494,7 +495,7 @@ def perform_create(self, serializer): image.save() -class ProjectCodeList(ListCreateAPI): +class ProjectCodeList(DataExportViewMixin, ListCreateAPI): """List view for all project codes.""" queryset = common.models.ProjectCode.objects.all() @@ -515,7 +516,7 @@ class ProjectCodeDetail(RetrieveUpdateDestroyAPI): permission_classes = [permissions.IsAuthenticated, IsStaffOrReadOnly] -class CustomUnitList(ListCreateAPI): +class CustomUnitList(DataExportViewMixin, ListCreateAPI): """List view for custom units.""" queryset = common.models.CustomUnit.objects.all() diff --git a/src/backend/InvenTree/common/migrations/0018_projectcode.py b/src/backend/InvenTree/common/migrations/0018_projectcode.py index 6ce6184ffb4d..544007012a21 100644 --- a/src/backend/InvenTree/common/migrations/0018_projectcode.py +++ b/src/backend/InvenTree/common/migrations/0018_projectcode.py @@ -17,5 +17,8 @@ class Migration(migrations.Migration): ('code', models.CharField(help_text='Unique project code', max_length=50, unique=True, verbose_name='Project Code')), ('description', models.CharField(blank=True, help_text='Project description', max_length=200, verbose_name='Description')), ], + options={ + 'verbose_name': 'Project Code', + }, ), ] diff --git a/src/backend/InvenTree/common/migrations/0020_customunit.py b/src/backend/InvenTree/common/migrations/0020_customunit.py index 500d34c68344..2c27252cf616 100644 --- a/src/backend/InvenTree/common/migrations/0020_customunit.py +++ b/src/backend/InvenTree/common/migrations/0020_customunit.py @@ -18,5 +18,8 @@ class Migration(migrations.Migration): ('symbol', models.CharField(blank=True, help_text='Optional unit symbol', max_length=10, unique=True, verbose_name='Symbol')), ('definition', models.CharField(help_text='Unit definition', max_length=50, verbose_name='Definition')), ], + options={ + 'verbose_name': 'Custom Unit', + }, ), ] diff --git a/src/backend/InvenTree/common/models.py b/src/backend/InvenTree/common/models.py index e7a3827c8e7b..791d36e68472 100644 --- a/src/backend/InvenTree/common/models.py +++ b/src/backend/InvenTree/common/models.py @@ -116,6 +116,11 @@ def __call__(self, value): class ProjectCode(InvenTree.models.InvenTreeMetadataModel): """A ProjectCode is a unique identifier for a project.""" + class Meta: + """Class options for the ProjectCode model.""" + + verbose_name = _('Project Code') + @staticmethod def get_api_url(): """Return the API URL for this model.""" @@ -3027,6 +3032,11 @@ class CustomUnit(models.Model): https://pint.readthedocs.io/en/stable/advanced/defining.html """ + class Meta: + """Class meta options.""" + + verbose_name = _('Custom Unit') + def fmt_string(self): """Construct a unit definition string e.g. 'dog_year = 52 * day = dy'.""" fmt = f'{self.name} = {self.definition}' diff --git a/src/backend/InvenTree/common/serializers.py b/src/backend/InvenTree/common/serializers.py index 0e0c7d18503b..45b681cb195c 100644 --- a/src/backend/InvenTree/common/serializers.py +++ b/src/backend/InvenTree/common/serializers.py @@ -14,6 +14,8 @@ import common.models as common_models import common.validators +from importer.mixins import DataImportExportSerializerMixin +from importer.registry import register_importer from InvenTree.helpers import get_objectreference from InvenTree.helpers_model import construct_absolute_url from InvenTree.serializers import ( @@ -293,7 +295,8 @@ class Meta: image = InvenTreeImageSerializerField(required=True) -class ProjectCodeSerializer(InvenTreeModelSerializer): +@register_importer() +class ProjectCodeSerializer(DataImportExportSerializerMixin, InvenTreeModelSerializer): """Serializer for the ProjectCode model.""" class Meta: @@ -341,7 +344,8 @@ def get_is_plugin(self, obj) -> bool: return obj.app_label in plugin_registry.installed_apps -class CustomUnitSerializer(InvenTreeModelSerializer): +@register_importer() +class CustomUnitSerializer(DataImportExportSerializerMixin, InvenTreeModelSerializer): """DRF serializer for CustomUnit model.""" class Meta: diff --git a/src/backend/InvenTree/part/api.py b/src/backend/InvenTree/part/api.py index 6d088f661f33..efe2a803a000 100644 --- a/src/backend/InvenTree/part/api.py +++ b/src/backend/InvenTree/part/api.py @@ -320,7 +320,7 @@ def get_queryset(self, *args, **kwargs): return queryset -class CategoryParameterList(ListCreateAPI): +class CategoryParameterList(DataExportViewMixin, ListCreateAPI): """API endpoint for accessing a list of PartCategoryParameterTemplate objects. - GET: Return a list of PartCategoryParameterTemplate objects @@ -375,7 +375,7 @@ class PartSalePriceDetail(RetrieveUpdateDestroyAPI): serializer_class = part_serializers.PartSalePriceSerializer -class PartSalePriceList(ListCreateAPI): +class PartSalePriceList(DataExportViewMixin, ListCreateAPI): """API endpoint for list view of PartSalePriceBreak model.""" queryset = PartSellPriceBreak.objects.all() @@ -394,7 +394,7 @@ class PartInternalPriceDetail(RetrieveUpdateDestroyAPI): serializer_class = part_serializers.PartInternalPriceSerializer -class PartInternalPriceList(ListCreateAPI): +class PartInternalPriceList(DataExportViewMixin, ListCreateAPI): """API endpoint for list view of PartInternalPriceBreak model.""" queryset = PartInternalPriceBreak.objects.all() @@ -470,7 +470,7 @@ class PartTestTemplateDetail(PartTestTemplateMixin, RetrieveUpdateDestroyAPI): pass -class PartTestTemplateList(PartTestTemplateMixin, ListCreateAPI): +class PartTestTemplateList(PartTestTemplateMixin, DataExportViewMixin, ListCreateAPI): """API endpoint for listing (and creating) a PartTestTemplate.""" filterset_class = PartTestTemplateFilter @@ -1518,7 +1518,9 @@ def get_queryset(self, *args, **kwargs): return queryset -class PartParameterTemplateList(PartParameterTemplateMixin, ListCreateAPI): +class PartParameterTemplateList( + PartParameterTemplateMixin, DataExportViewMixin, ListCreateAPI +): """API endpoint for accessing a list of PartParameterTemplate objects. - GET: Return list of PartParameterTemplate objects @@ -1599,7 +1601,7 @@ def filter_part(self, queryset, name, part): return queryset.filter(part=part) -class PartParameterList(PartParameterAPIMixin, ListCreateAPI): +class PartParameterList(PartParameterAPIMixin, DataExportViewMixin, ListCreateAPI): """API endpoint for accessing a list of PartParameter objects. - GET: Return list of PartParameter objects @@ -1827,7 +1829,7 @@ def get_queryset(self, *args, **kwargs): return queryset -class BomList(BomMixin, ListCreateDestroyAPIView): +class BomList(BomMixin, DataExportViewMixin, ListCreateDestroyAPIView): """API endpoint for accessing a list of BomItem objects. - GET: Return list of BomItem objects diff --git a/src/backend/InvenTree/part/serializers.py b/src/backend/InvenTree/part/serializers.py index 9a7ece80f82d..d0bbd953ca96 100644 --- a/src/backend/InvenTree/part/serializers.py +++ b/src/backend/InvenTree/part/serializers.py @@ -197,7 +197,10 @@ def annotate_queryset(queryset): return queryset.annotate(results=SubqueryCount('test_results')) -class PartSalePriceSerializer(InvenTree.serializers.InvenTreeModelSerializer): +@register_importer() +class PartSalePriceSerializer( + DataImportExportSerializerMixin, InvenTree.serializers.InvenTreeModelSerializer +): """Serializer for sale prices for Part model.""" class Meta: @@ -1699,8 +1702,9 @@ def annotate_queryset(queryset): return queryset +@register_importer() class CategoryParameterTemplateSerializer( - InvenTree.serializers.InvenTreeModelSerializer + DataImportExportSerializerMixin, InvenTree.serializers.InvenTreeModelSerializer ): """Serializer for the PartCategoryParameterTemplate model.""" diff --git a/src/frontend/src/components/importer/ImportDataSelector.tsx b/src/frontend/src/components/importer/ImportDataSelector.tsx index e546398fecbc..46a337826769 100644 --- a/src/frontend/src/components/importer/ImportDataSelector.tsx +++ b/src/frontend/src/components/importer/ImportDataSelector.tsx @@ -227,9 +227,15 @@ export default function ImporterDataSelector({ let errors: string[] = []; for (const k of Object.keys(row.errors)) { - row.errors[k].forEach((e: string) => { - errors.push(`${k}: ${e}`); - }); + if (row.errors[k]) { + if (Array.isArray(row.errors[k])) { + row.errors[k].forEach((e: string) => { + errors.push(`${k}: ${e}`); + }); + } else { + errors.push(row.errors[k].toString()); + } + } } return errors; diff --git a/src/frontend/src/tables/InvenTreeTable.tsx b/src/frontend/src/tables/InvenTreeTable.tsx index cbc3eda0386d..1bdbc04acc8e 100644 --- a/src/frontend/src/tables/InvenTreeTable.tsx +++ b/src/frontend/src/tables/InvenTreeTable.tsx @@ -604,12 +604,6 @@ export function InvenTreeTable({ - {tableProps.enableDownload && ( - - )} {tableProps.enableUpload && } ({ )} + {tableProps.enableDownload && ( + + )} diff --git a/src/frontend/src/tables/part/PartParameterTable.tsx b/src/frontend/src/tables/part/PartParameterTable.tsx index fd85551b3412..c759988698cc 100644 --- a/src/frontend/src/tables/part/PartParameterTable.tsx +++ b/src/frontend/src/tables/part/PartParameterTable.tsx @@ -184,6 +184,7 @@ export function PartParameterTable({ partId }: { partId: any }) { columns={tableColumns} props={{ rowActions: rowActions, + enableDownload: true, tableActions: tableActions, tableFilters: [ { diff --git a/src/frontend/src/tables/part/PartParameterTemplateTable.tsx b/src/frontend/src/tables/part/PartParameterTemplateTable.tsx index 5d86ae1e4358..e351eedc12e6 100644 --- a/src/frontend/src/tables/part/PartParameterTemplateTable.tsx +++ b/src/frontend/src/tables/part/PartParameterTemplateTable.tsx @@ -157,7 +157,8 @@ export default function PartParameterTemplateTable() { props={{ rowActions: rowActions, tableFilters: tableFilters, - tableActions: tableActions + tableActions: tableActions, + enableDownload: true }} /> diff --git a/src/frontend/src/tables/settings/CustomUnitsTable.tsx b/src/frontend/src/tables/settings/CustomUnitsTable.tsx index 3d0f2341c368..a26fc2152d30 100644 --- a/src/frontend/src/tables/settings/CustomUnitsTable.tsx +++ b/src/frontend/src/tables/settings/CustomUnitsTable.tsx @@ -116,7 +116,8 @@ export default function CustomUnitsTable() { columns={columns} props={{ rowActions: rowActions, - tableActions: tableActions + tableActions: tableActions, + enableDownload: true }} /> diff --git a/src/frontend/src/tables/settings/ProjectCodeTable.tsx b/src/frontend/src/tables/settings/ProjectCodeTable.tsx index 8a20d161f936..b1d9b1694d50 100644 --- a/src/frontend/src/tables/settings/ProjectCodeTable.tsx +++ b/src/frontend/src/tables/settings/ProjectCodeTable.tsx @@ -105,7 +105,8 @@ export default function ProjectCodeTable() { columns={columns} props={{ rowActions: rowActions, - tableActions: tableActions + tableActions: tableActions, + enableDownload: true }} /> From f9b13b6464756a6046a1884b039e2b000cfc15b0 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 10:50:45 +0000 Subject: [PATCH 176/190] Fix verbose name --- src/backend/InvenTree/part/migrations/0049_partsellpricebreak.py | 1 + src/backend/InvenTree/part/models.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/backend/InvenTree/part/migrations/0049_partsellpricebreak.py b/src/backend/InvenTree/part/migrations/0049_partsellpricebreak.py index 1d49dcbfac80..8332d353af7f 100644 --- a/src/backend/InvenTree/part/migrations/0049_partsellpricebreak.py +++ b/src/backend/InvenTree/part/migrations/0049_partsellpricebreak.py @@ -24,6 +24,7 @@ class Migration(migrations.Migration): ('part', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='salepricebreaks', to='part.Part')), ], options={ + 'verbose_name': 'Part Sale Price Break', 'unique_together': {('part', 'quantity')}, }, ), diff --git a/src/backend/InvenTree/part/models.py b/src/backend/InvenTree/part/models.py index af913ce33f33..34c431406568 100644 --- a/src/backend/InvenTree/part/models.py +++ b/src/backend/InvenTree/part/models.py @@ -3288,6 +3288,7 @@ class PartSellPriceBreak(common.models.PriceBreak): class Meta: """Metaclass providing extra model definition.""" + verbose_name = _('Part Sale Price Break') unique_together = ('part', 'quantity') @staticmethod From 394f56c6b20fee9eb759ab0f5ec5921b20d6e400 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Thu, 4 Jul 2024 11:23:25 +0000 Subject: [PATCH 177/190] Import rows as a bulk db operation --- src/backend/InvenTree/importer/models.py | 54 +++++++++++++++--------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index 8f3528c920d4..e4902327083a 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -229,7 +229,7 @@ def trigger_data_import(self): self.status = DataImportStatusCode.IMPORTING.value self.save() - offload_task(importer.tasks.import_data, self.pk, force_async=True) + offload_task(importer.tasks.import_data, self.pk) def import_data(self): """Perform the data import process for this session.""" @@ -245,25 +245,34 @@ def import_data(self): headers = df.headers - row_objects = [] + imported_rows = [] + + field_mapping = self.field_mapping + available_fields = self.available_fields() # Iterate through each "row" in the data file, and create a new DataImportRow object for idx, row in enumerate(df): row_data = dict(zip(headers, row)) - row_objects.append( - importer.models.DataImportRow( - session=self, row_data=row_data, row_index=idx - ) + # Skip completely empty rows + if not any(row_data.values()): + continue + + row = importer.models.DataImportRow( + session=self, row_data=row_data, row_index=idx + ) + + row.extract_data( + field_mapping=field_mapping, + available_fields=available_fields, + commit=False, ) + row.validate(commit=False) - # Create the DataImportRow objects - importer.models.DataImportRow.objects.bulk_create(row_objects) + imported_rows.append(row) - # Set initial data and errors for each row - for row in self.rows.all(): - row.extract_data(field_mapping=self.field_mapping) - row.validate() + # Perform database writes as a single operation + importer.models.DataImportRow.objects.bulk_create(imported_rows) # Mark the import task as "PROCESSING" self.status = DataImportStatusCode.PROCESSING.value @@ -471,12 +480,15 @@ def save(self, *args, **kwargs): complete = models.BooleanField(default=False, verbose_name=_('Complete')) - def extract_data(self, field_mapping: dict = None): + def extract_data( + self, available_fields: dict = None, field_mapping: dict = None, commit=True + ): """Extract row data from the provided data dictionary.""" if not field_mapping: field_mapping = self.session.field_mapping - available_fields = self.session.available_fields() + if not available_fields: + available_fields = self.session.available_fields() override_values = self.session.field_overrides or {} default_values = self.session.field_defaults or {} @@ -513,7 +525,9 @@ def extract_data(self, field_mapping: dict = None): data[field] = value self.data = data - self.save() + + if commit: + self.save() def serializer_data(self): """Construct data object to be sent to the serializer. @@ -521,11 +535,13 @@ def serializer_data(self): - If available, we use the "default" values provided by the import session - If available, we use the "override" values provided by the import session """ - session_defaults = self.session.field_defaults or {} - session_overrides = self.session.field_overrides or {} + data = self.session.field_defaults or {} + + if self.data: + data.update(self.data) - # Construct data - data = {**session_defaults, **self.data, **session_overrides} + if self.session.field_overrides: + data.update(self.session.field_overrides) return data From 39963c6b8575b2f7e18e850a49aef50490307a39 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 06:07:04 +0000 Subject: [PATCH 178/190] Enable download for PartCategoryTemplateTable --- src/frontend/src/tables/part/PartCategoryTemplateTable.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/frontend/src/tables/part/PartCategoryTemplateTable.tsx b/src/frontend/src/tables/part/PartCategoryTemplateTable.tsx index e44b756df1ea..161722671673 100644 --- a/src/frontend/src/tables/part/PartCategoryTemplateTable.tsx +++ b/src/frontend/src/tables/part/PartCategoryTemplateTable.tsx @@ -147,7 +147,8 @@ export default function PartCategoryTemplateTable({}: {}) { props={{ rowActions: rowActions, tableFilters: tableFilters, - tableActions: tableActions + tableActions: tableActions, + enableDownload: true }} /> From 17f7cc388b78e25b82124e9d577e6533c1f931e7 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 06:19:27 +0000 Subject: [PATCH 179/190] Update stock item export --- src/backend/InvenTree/stock/serializers.py | 16 ++++++++++++++++ src/backend/InvenTree/stock/test_api.py | 8 ++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/backend/InvenTree/stock/serializers.py b/src/backend/InvenTree/stock/serializers.py index 26f393f0b93f..63e79b4981fe 100644 --- a/src/backend/InvenTree/stock/serializers.py +++ b/src/backend/InvenTree/stock/serializers.py @@ -352,11 +352,13 @@ class Meta: 'is_building', 'link', 'location', + 'location_name', 'location_detail', 'location_path', 'notes', 'owner', 'packaging', + 'parent', 'part', 'part_detail', 'purchase_order', @@ -441,6 +443,17 @@ def __init__(self, *args, **kwargs): label=_('Part'), ) + parent = serializers.PrimaryKeyRelatedField( + many=False, + read_only=True, + label=_('Parent Item'), + help_text=_('Parent stock item'), + ) + + location_name = serializers.CharField( + source='location.name', read_only=True, label=_('Location Name') + ) + location_path = serializers.ListField( child=serializers.DictField(), source='location.get_path', read_only=True ) @@ -486,6 +499,7 @@ def annotate_queryset(queryset): ) ).prefetch_related(None), ), + 'parent', 'part__category', 'part__pricing_data', 'supplier_part', @@ -555,9 +569,11 @@ def annotate_queryset(queryset): read_only=True, ) part_detail = PartBriefSerializer(source='part', many=False, read_only=True) + location_detail = LocationBriefSerializer( source='location', many=False, read_only=True ) + tests = StockItemTestResultSerializer( source='test_results', many=True, read_only=True ) diff --git a/src/backend/InvenTree/stock/test_api.py b/src/backend/InvenTree/stock/test_api.py index 7848fd823443..4b8471cf8bcd 100644 --- a/src/backend/InvenTree/stock/test_api.py +++ b/src/backend/InvenTree/stock/test_api.py @@ -765,11 +765,11 @@ def test_export(self): # Expected headers headers = [ - 'Part ID', - 'Customer ID', - 'Location ID', + 'Part', + 'Customer', + 'Stock Location', 'Location Name', - 'Parent ID', + 'Parent Item', 'Quantity', 'Status', ] From 1b8adaeca773fd08b940dfb8521661f256a90901 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 06:42:02 +0000 Subject: [PATCH 180/190] Updates for unit tests --- src/backend/InvenTree/InvenTree/unit_test.py | 2 +- .../migrations/0023_auto_20240602_1332.py | 11 +++++--- src/backend/InvenTree/common/tests.py | 2 +- src/backend/InvenTree/part/serializers.py | 9 +++++-- src/backend/InvenTree/part/test_api.py | 27 ++++++++++--------- 5 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/unit_test.py b/src/backend/InvenTree/InvenTree/unit_test.py index 583930f79059..d7ce3d7cfba2 100644 --- a/src/backend/InvenTree/InvenTree/unit_test.py +++ b/src/backend/InvenTree/InvenTree/unit_test.py @@ -412,7 +412,7 @@ def download_file( # Extract filename disposition = response.headers['Content-Disposition'] - result = re.search(r'attachment; filename="([\w.]+)"', disposition) + result = re.search(r'attachment; filename="([\w\d\-.]+)"', disposition) fn = result.groups()[0] diff --git a/src/backend/InvenTree/common/migrations/0023_auto_20240602_1332.py b/src/backend/InvenTree/common/migrations/0023_auto_20240602_1332.py index a3b964cbaab7..66b78f64760c 100644 --- a/src/backend/InvenTree/common/migrations/0023_auto_20240602_1332.py +++ b/src/backend/InvenTree/common/migrations/0023_auto_20240602_1332.py @@ -1,5 +1,6 @@ # Generated by Django 4.2.12 on 2024-06-02 13:32 +from django.conf import settings from django.db import migrations from moneyed import CURRENCIES @@ -47,16 +48,20 @@ def set_currencies(apps, schema_editor): return value = ','.join(valid_codes) - print(f"Found existing currency codes:", value) + + if not settings.TESTING: + print(f"Found existing currency codes:", value) setting = InvenTreeSetting.objects.filter(key=key).first() if setting: - print(f"- Updating existing setting for currency codes") + if not settings.TESTING: + print(f"- Updating existing setting for currency codes") setting.value = value setting.save() else: - print(f"- Creating new setting for currency codes") + if not settings.TESTING: + print(f"- Creating new setting for currency codes") setting = InvenTreeSetting(key=key, value=value) setting.save() diff --git a/src/backend/InvenTree/common/tests.py b/src/backend/InvenTree/common/tests.py index f936fddfb9c2..3d7f9c8a5b05 100644 --- a/src/backend/InvenTree/common/tests.py +++ b/src/backend/InvenTree/common/tests.py @@ -1376,7 +1376,7 @@ def test_duplicate_code(self): ) self.assertIn( - 'project code with this Project Code already exists', + 'Project code with this Project Code already exists', str(response.data['code']), ) diff --git a/src/backend/InvenTree/part/serializers.py b/src/backend/InvenTree/part/serializers.py index d0bbd953ca96..97ce008a9bc2 100644 --- a/src/backend/InvenTree/part/serializers.py +++ b/src/backend/InvenTree/part/serializers.py @@ -615,6 +615,7 @@ class Meta: 'category', 'category_detail', 'category_path', + 'category_name', 'component', 'creation_date', 'creation_user', @@ -829,6 +830,10 @@ def get_starred(self, part) -> bool: child=serializers.DictField(), source='category.get_path', read_only=True ) + category_name = serializers.CharField( + source='category.name', read_only=True, label=_('Category Name') + ) + responsible = serializers.PrimaryKeyRelatedField( queryset=users.models.Owner.objects.all(), required=False, @@ -843,8 +848,8 @@ def get_starred(self, part) -> bool: # Annotated fields allocated_to_build_orders = serializers.FloatField(read_only=True) allocated_to_sales_orders = serializers.FloatField(read_only=True) - building = serializers.FloatField(read_only=True) - in_stock = serializers.FloatField(read_only=True) + building = serializers.FloatField(read_only=True, label=_('Building')) + in_stock = serializers.FloatField(read_only=True, label=_('In Stock')) ordering = serializers.FloatField(read_only=True, label=_('On Order')) required_for_build_orders = serializers.IntegerField(read_only=True) required_for_sales_orders = serializers.IntegerField(read_only=True) diff --git a/src/backend/InvenTree/part/test_api.py b/src/backend/InvenTree/part/test_api.py index 6fb726541093..078c53f58688 100644 --- a/src/backend/InvenTree/part/test_api.py +++ b/src/backend/InvenTree/part/test_api.py @@ -1033,25 +1033,26 @@ def test_part_download(self): url = reverse('api-part-list') required_cols = [ - 'Part ID', - 'Part Name', - 'Part Description', - 'In Stock', + 'ID', + 'Name', + 'Description', + 'Total Stock', 'Category Name', 'Keywords', - 'Template', + 'Is Template', 'Virtual', 'Trackable', 'Active', 'Notes', - 'creation_date', + 'Creation Date', + 'On Order', + 'In Stock', + 'Link', ] excluded_cols = ['lft', 'rght', 'level', 'tree_id', 'metadata'] - with self.download_file( - url, {'export': 'csv'}, expected_fn='InvenTree_Parts.csv' - ) as file: + with self.download_file(url, {'export': 'csv'}) as file: data = self.process_csv( file, excluded_cols=excluded_cols, @@ -1060,13 +1061,13 @@ def test_part_download(self): ) for row in data: - part = Part.objects.get(pk=row['Part ID']) + part = Part.objects.get(pk=row['ID']) if part.IPN: self.assertEqual(part.IPN, row['IPN']) - self.assertEqual(part.name, row['Part Name']) - self.assertEqual(part.description, row['Part Description']) + self.assertEqual(part.name, row['Name']) + self.assertEqual(part.description, row['Description']) if part.category: self.assertEqual(part.category.name, row['Category Name']) @@ -2936,7 +2937,7 @@ def test_choices(self): options = response.data['actions']['PUT'] self.assertTrue(options['pk']['read_only']) - self.assertTrue(options['pk']['required']) + self.assertFalse(options['pk']['required']) self.assertEqual(options['part']['api_url'], '/api/part/') self.assertTrue(options['test_name']['required']) self.assertFalse(options['test_name']['read_only']) From 5ef7a56e295a8c16a2f0fe461945c74def12ec08 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 07:00:20 +0000 Subject: [PATCH 181/190] Remove xls format for now - Causes some bug in tablib - Surely xlsx is OK? --- src/backend/InvenTree/InvenTree/helpers.py | 2 +- src/backend/InvenTree/order/test_api.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/helpers.py b/src/backend/InvenTree/InvenTree/helpers.py index 88c7d4e1523b..86eb69ab1005 100644 --- a/src/backend/InvenTree/InvenTree/helpers.py +++ b/src/backend/InvenTree/InvenTree/helpers.py @@ -430,7 +430,7 @@ def MakeBarcode(cls_name, object_pk: int, object_data=None, **kwargs): def GetExportFormats(): """Return a list of allowable file formats for importing or exporting tabular data.""" - return ['csv', 'tsv', 'xls', 'xlsx', 'json', 'yaml'] + return ['csv', 'tsv', 'xlsx', 'json', 'yaml'] def DownloadFile( diff --git a/src/backend/InvenTree/order/test_api.py b/src/backend/InvenTree/order/test_api.py index 5fb5fb560de7..2f2583e54b5c 100644 --- a/src/backend/InvenTree/order/test_api.py +++ b/src/backend/InvenTree/order/test_api.py @@ -1641,11 +1641,7 @@ def test_download_xls(self): # Download .xls file with self.download_file( - url, - {'export': 'xls'}, - expected_code=200, - expected_fn='InvenTree_SalesOrders.xls', - decode=False, + url, {'export': 'xlsx'}, expected_code=200, decode=False ) as file: self.assertIsInstance(file, io.BytesIO) From 3713fb9851cebccc39a9abc45fdc5403a3280f26 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 07:10:49 +0000 Subject: [PATCH 182/190] More unit test updates --- src/backend/InvenTree/order/test_api.py | 35 ++++++++++--------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/src/backend/InvenTree/order/test_api.py b/src/backend/InvenTree/order/test_api.py index 2f2583e54b5c..e9c25d661995 100644 --- a/src/backend/InvenTree/order/test_api.py +++ b/src/backend/InvenTree/order/test_api.py @@ -1635,8 +1635,8 @@ def test_download_fail(self): with self.assertRaises(ValueError): self.download_file(url, {}, expected_code=200) - def test_download_xls(self): - """Test xls file download.""" + def test_download_xlsx(self): + """Test xlsx file download.""" url = reverse('api-so-list') # Download .xls file @@ -1650,25 +1650,22 @@ def test_download_csv(self): url = reverse('api-so-list') required_cols = [ - 'line_items', - 'id', - 'reference', - 'customer', - 'status', - 'shipment_date', - 'notes', - 'description', + 'Line Items', + 'ID', + 'Reference', + 'Customer', + 'Order Status', + 'Shipment Date', + 'Description', + 'Project Code', + 'Responsible', ] excluded_cols = ['metadata'] # Download .xls file with self.download_file( - url, - {'export': 'csv'}, - expected_code=200, - expected_fn='InvenTree_SalesOrders.csv', - decode=True, + url, {'export': 'csv'}, expected_code=200, decode=True ) as file: data = self.process_csv( file, @@ -1678,18 +1675,14 @@ def test_download_csv(self): ) for line in data: - order = models.SalesOrder.objects.get(pk=line['id']) + order = models.SalesOrder.objects.get(pk=line['ID']) self.assertEqual(line['description'], order.description) self.assertEqual(line['status'], str(order.status)) # Download only outstanding sales orders with self.download_file( - url, - {'export': 'tsv', 'outstanding': True}, - expected_code=200, - expected_fn='InvenTree_SalesOrders.tsv', - decode=True, + url, {'export': 'tsv', 'outstanding': True}, expected_code=200, decode=True ) as file: self.process_csv( file, From c6342dd70e122b83443671930ac5d25364ba54d6 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 10:06:58 +0000 Subject: [PATCH 183/190] Future proof migration --- src/backend/InvenTree/importer/migrations/0001_initial.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/backend/InvenTree/importer/migrations/0001_initial.py b/src/backend/InvenTree/importer/migrations/0001_initial.py index 13440c66f725..7a2403272865 100644 --- a/src/backend/InvenTree/importer/migrations/0001_initial.py +++ b/src/backend/InvenTree/importer/migrations/0001_initial.py @@ -5,6 +5,8 @@ from django.db import migrations, models import django.db.models.deletion import importer.validators +import InvenTree.helpers +from importer.status_codes import DataImportStatusCode class Migration(migrations.Migration): @@ -21,10 +23,10 @@ class Migration(migrations.Migration): fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('timestamp', models.DateTimeField(auto_now_add=True, verbose_name='Timestamp')), - ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=['csv', 'tsv', 'xls', 'xlsx', 'json', 'yaml']), importer.validators.validate_data_file], verbose_name='Data File')), + ('data_file', models.FileField(help_text='Data file to import', upload_to='import', validators=[django.core.validators.FileExtensionValidator(allowed_extensions=InvenTree.helpers.GetExportFormats()), importer.validators.validate_data_file], verbose_name='Data File')), ('columns', models.JSONField(blank=True, null=True, verbose_name='Columns')), ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), - ('status', models.PositiveIntegerField(choices=[(0, 'Initializing'), (10, 'Mapping Columns'), (20, 'Importing Data'), (30, 'Processing Data'), (40, 'Complete')], default=0, help_text='Import status')), + ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL.value, help_text='Import status')), ('field_defaults', models.JSONField(blank=True, null=True, validators=[importer.validators.validate_field_defaults], verbose_name='Field Defaults')), ('field_overrides', models.JSONField(blank=True, null=True, validators=[importer.validators.validate_field_defaults], verbose_name='Field Overrides')), ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), From 319e5bcdf60e586030dd45380317c8784c0a53bd Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 10:57:33 +0000 Subject: [PATCH 184/190] Updates --- src/backend/InvenTree/InvenTree/helpers.py | 2 +- .../InvenTree/static/script/inventree/inventree.js | 4 ---- .../InvenTree/templatetags/inventree_extras.py | 5 ++++- src/backend/InvenTree/InvenTree/unit_test.py | 2 +- src/backend/InvenTree/order/test_api.py | 8 ++++---- src/backend/InvenTree/part/test_bom_export.py | 10 +++++----- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/helpers.py b/src/backend/InvenTree/InvenTree/helpers.py index 86eb69ab1005..b4543adbbb34 100644 --- a/src/backend/InvenTree/InvenTree/helpers.py +++ b/src/backend/InvenTree/InvenTree/helpers.py @@ -430,7 +430,7 @@ def MakeBarcode(cls_name, object_pk: int, object_data=None, **kwargs): def GetExportFormats(): """Return a list of allowable file formats for importing or exporting tabular data.""" - return ['csv', 'tsv', 'xlsx', 'json', 'yaml'] + return ['csv', 'tsv', 'xlsx'] def DownloadFile( diff --git a/src/backend/InvenTree/InvenTree/static/script/inventree/inventree.js b/src/backend/InvenTree/InvenTree/static/script/inventree/inventree.js index db12de9d9dbb..0da91dbee11e 100644 --- a/src/backend/InvenTree/InvenTree/static/script/inventree/inventree.js +++ b/src/backend/InvenTree/InvenTree/static/script/inventree/inventree.js @@ -60,10 +60,6 @@ function exportFormatOptions() { value: 'tsv', display_name: 'TSV', }, - { - value: 'xls', - display_name: 'XLS', - }, { value: 'xlsx', display_name: 'XLSX', diff --git a/src/backend/InvenTree/InvenTree/templatetags/inventree_extras.py b/src/backend/InvenTree/InvenTree/templatetags/inventree_extras.py index 1ac050f7e6bc..ad476ba6711d 100644 --- a/src/backend/InvenTree/InvenTree/templatetags/inventree_extras.py +++ b/src/backend/InvenTree/InvenTree/templatetags/inventree_extras.py @@ -455,7 +455,10 @@ def get_user_color_theme(user): """Get current user color theme.""" from common.models import ColorTheme - if not user.is_authenticated: + try: + if not user.is_authenticated: + return 'default' + except Exception: return 'default' try: diff --git a/src/backend/InvenTree/InvenTree/unit_test.py b/src/backend/InvenTree/InvenTree/unit_test.py index d7ce3d7cfba2..94af1325c41d 100644 --- a/src/backend/InvenTree/InvenTree/unit_test.py +++ b/src/backend/InvenTree/InvenTree/unit_test.py @@ -417,7 +417,7 @@ def download_file( fn = result.groups()[0] if expected_fn is not None: - self.assertEqual(expected_fn, fn) + self.assertRegex(fn, expected_fn) if decode: # Decode data and return as StringIO file object diff --git a/src/backend/InvenTree/order/test_api.py b/src/backend/InvenTree/order/test_api.py index e9c25d661995..764919f51bf8 100644 --- a/src/backend/InvenTree/order/test_api.py +++ b/src/backend/InvenTree/order/test_api.py @@ -818,7 +818,7 @@ def test_download_csv(self): reverse('api-po-list'), {'export': 'csv'}, expected_code=200, - expected_fn='InvenTree_PurchaseOrders.csv', + expected_fn=r'InvenTree_PurchaseOrder_.+\.csv', ) as file: data = self.process_csv( file, @@ -840,7 +840,7 @@ def test_download_line_items(self): {'export': 'xlsx'}, decode=False, expected_code=200, - expected_fn='InvenTree_PurchaseOrderItems.xlsx', + expected_fn=r'InvenTree_PurchaseOrderItem.+\.xlsx', ) as file: self.assertIsInstance(file, io.BytesIO) @@ -1473,13 +1473,13 @@ def test_export(self): order.save() # Download file, check we get a 200 response - for fmt in ['csv', 'xls', 'xlsx']: + for fmt in ['csv', 'xlsx', 'tsv']: self.download_file( reverse('api-so-list'), {'export': fmt}, decode=True if fmt == 'csv' else False, expected_code=200, - expected_fn=f'InvenTree_SalesOrders.{fmt}', + expected_fn=r'InvenTree_SalesOrder_.+', ) def test_sales_order_complete(self): diff --git a/src/backend/InvenTree/part/test_bom_export.py b/src/backend/InvenTree/part/test_bom_export.py index c1abe0de398b..6913c4394496 100644 --- a/src/backend/InvenTree/part/test_bom_export.py +++ b/src/backend/InvenTree/part/test_bom_export.py @@ -29,11 +29,11 @@ def test_bom_template(self): url = reverse('api-bom-upload-template') # Download an XLS template - response = self.client.get(url, data={'format': 'xls'}) + response = self.client.get(url, data={'format': 'xlsx'}) self.assertEqual(response.status_code, 200) self.assertEqual( response.headers['Content-Disposition'], - 'attachment; filename="InvenTree_BOM_Template.xls"', + 'attachment; filename="InvenTree_BOM_Template.xlsx"', ) # Return a simple CSV template @@ -134,10 +134,10 @@ def test_export_csv(self): for header in headers: self.assertIn(header, expected) - def test_export_xls(self): - """Test BOM download in XLS format.""" + def test_export_xlsx(self): + """Test BOM download in XLSX format.""" params = { - 'format': 'xls', + 'format': 'xlsx', 'cascade': True, 'parameter_data': True, 'stock_data': True, From d8126ffb0b90ec1ef8f26b8ed89d64dc8a5bb91b Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 11:34:15 +0000 Subject: [PATCH 185/190] unit tests --- src/backend/InvenTree/InvenTree/helpers.py | 2 +- src/backend/InvenTree/build/serializers.py | 3 +++ src/backend/InvenTree/build/test_api.py | 30 +++++++++++----------- src/backend/InvenTree/common/tests.py | 2 +- src/backend/InvenTree/order/test_api.py | 10 ++++---- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/backend/InvenTree/InvenTree/helpers.py b/src/backend/InvenTree/InvenTree/helpers.py index b4543adbbb34..ae72f59578e0 100644 --- a/src/backend/InvenTree/InvenTree/helpers.py +++ b/src/backend/InvenTree/InvenTree/helpers.py @@ -430,7 +430,7 @@ def MakeBarcode(cls_name, object_pk: int, object_data=None, **kwargs): def GetExportFormats(): """Return a list of allowable file formats for importing or exporting tabular data.""" - return ['csv', 'tsv', 'xlsx'] + return ['csv', 'xlsx', 'tsv', 'json'] def DownloadFile( diff --git a/src/backend/InvenTree/build/serializers.py b/src/backend/InvenTree/build/serializers.py index 1e8d308b955f..3efd581bbf75 100644 --- a/src/backend/InvenTree/build/serializers.py +++ b/src/backend/InvenTree/build/serializers.py @@ -51,6 +51,7 @@ class Meta: 'destination', 'parent', 'part', + 'part_name', 'part_detail', 'project_code', 'project_code_detail', @@ -85,6 +86,8 @@ class Meta: part_detail = PartBriefSerializer(source='part', many=False, read_only=True) + part_name = serializers.CharField(source='part.name', read_only=True, label=_('Part Name')) + quantity = InvenTreeDecimalField() overdue = serializers.BooleanField(required=False, read_only=True) diff --git a/src/backend/InvenTree/build/test_api.py b/src/backend/InvenTree/build/test_api.py index b240521db694..435db8367ea0 100644 --- a/src/backend/InvenTree/build/test_api.py +++ b/src/backend/InvenTree/build/test_api.py @@ -564,16 +564,16 @@ def test_create_delete_output(self): def test_download_build_orders(self): """Test that we can download a list of build orders via the API""" required_cols = [ - 'reference', - 'status', - 'completed', - 'batch', - 'notes', - 'title', - 'part', - 'part_name', - 'id', - 'quantity', + 'Reference', + 'Build Status', + 'Completed items', + 'Batch Code', + 'Notes', + 'Description', + 'Part', + 'Part Name', + 'ID', + 'Quantity', ] excluded_cols = [ @@ -597,13 +597,13 @@ def test_download_build_orders(self): for row in data: - build = Build.objects.get(pk=row['id']) + build = Build.objects.get(pk=row['ID']) - self.assertEqual(str(build.part.pk), row['part']) - self.assertEqual(build.part.full_name, row['part_name']) + self.assertEqual(str(build.part.pk), row['Part']) + self.assertEqual(build.part.name, row['Part Name']) - self.assertEqual(build.reference, row['reference']) - self.assertEqual(build.title, row['title']) + self.assertEqual(build.reference, row['Reference']) + self.assertEqual(build.title, row['Description']) class BuildAllocationTest(BuildAPITest): diff --git a/src/backend/InvenTree/common/tests.py b/src/backend/InvenTree/common/tests.py index 3d7f9c8a5b05..9c3dea87d721 100644 --- a/src/backend/InvenTree/common/tests.py +++ b/src/backend/InvenTree/common/tests.py @@ -1376,7 +1376,7 @@ def test_duplicate_code(self): ) self.assertIn( - 'Project code with this Project Code already exists', + 'Project Code with this Project Code already exists', str(response.data['code']), ) diff --git a/src/backend/InvenTree/order/test_api.py b/src/backend/InvenTree/order/test_api.py index 764919f51bf8..fcd396f24cf4 100644 --- a/src/backend/InvenTree/order/test_api.py +++ b/src/backend/InvenTree/order/test_api.py @@ -830,8 +830,8 @@ def test_download_csv(self): for row in data: order = models.PurchaseOrder.objects.get(pk=row['id']) - self.assertEqual(order.description, row['description']) - self.assertEqual(order.reference, row['reference']) + self.assertEqual(order.description, row['Description']) + self.assertEqual(order.reference, row['Reference']) def test_download_line_items(self): """Test that the PurchaseOrderLineItems can be downloaded to a file.""" @@ -840,7 +840,7 @@ def test_download_line_items(self): {'export': 'xlsx'}, decode=False, expected_code=200, - expected_fn=r'InvenTree_PurchaseOrderItem.+\.xlsx', + expected_fn=r'InvenTree_PurchaseOrderLineItem.+\.xlsx', ) as file: self.assertIsInstance(file, io.BytesIO) @@ -1677,8 +1677,8 @@ def test_download_csv(self): for line in data: order = models.SalesOrder.objects.get(pk=line['ID']) - self.assertEqual(line['description'], order.description) - self.assertEqual(line['status'], str(order.status)) + self.assertEqual(line['Description'], order.description) + self.assertEqual(line['Order Status'], str(order.status)) # Download only outstanding sales orders with self.download_file( From 732ec5b0f344712848545bb86ec2ac8312c9d203 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 12:41:23 +0000 Subject: [PATCH 186/190] Unit test fix --- src/backend/InvenTree/order/test_api.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/backend/InvenTree/order/test_api.py b/src/backend/InvenTree/order/test_api.py index fcd396f24cf4..a2257755f205 100644 --- a/src/backend/InvenTree/order/test_api.py +++ b/src/backend/InvenTree/order/test_api.py @@ -793,14 +793,14 @@ class PurchaseOrderDownloadTest(OrderTest): """Unit tests for downloading PurchaseOrder data via the API endpoint.""" required_cols = [ - 'id', - 'line_items', - 'description', - 'issue_date', - 'notes', - 'reference', - 'status', - 'supplier_reference', + 'ID', + 'Line Items', + 'Description', + 'Issue Date', + 'Order Currency', + 'Reference', + 'Order Status', + 'Supplier Reference', ] excluded_cols = ['metadata'] @@ -828,7 +828,7 @@ def test_download_csv(self): ) for row in data: - order = models.PurchaseOrder.objects.get(pk=row['id']) + order = models.PurchaseOrder.objects.get(pk=row['ID']) self.assertEqual(order.description, row['Description']) self.assertEqual(order.reference, row['Reference']) From 43f7dd6d8bc0c091dd66d4fdd7154fcedccffafd Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 22:40:14 +0000 Subject: [PATCH 187/190] Remove 'field_overrides' - field_defaults will suffice --- .../importer/migrations/0001_initial.py | 3 +-- src/backend/InvenTree/importer/models.py | 26 ++----------------- src/backend/InvenTree/importer/serializers.py | 1 - 3 files changed, 3 insertions(+), 27 deletions(-) diff --git a/src/backend/InvenTree/importer/migrations/0001_initial.py b/src/backend/InvenTree/importer/migrations/0001_initial.py index 7a2403272865..0572c1670450 100644 --- a/src/backend/InvenTree/importer/migrations/0001_initial.py +++ b/src/backend/InvenTree/importer/migrations/0001_initial.py @@ -28,8 +28,7 @@ class Migration(migrations.Migration): ('model_type', models.CharField(max_length=100, validators=[importer.validators.validate_importer_model_type])), ('status', models.PositiveIntegerField(choices=DataImportStatusCode.items(), default=DataImportStatusCode.INITIAL.value, help_text='Import status')), ('field_defaults', models.JSONField(blank=True, null=True, validators=[importer.validators.validate_field_defaults], verbose_name='Field Defaults')), - ('field_overrides', models.JSONField(blank=True, null=True, validators=[importer.validators.validate_field_defaults], verbose_name='Field Overrides')), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='User')), + ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, verbose_name='User')), ], ), migrations.CreateModel( diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index e4902327083a..f88615555347 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -32,7 +32,6 @@ class DataImportSession(models.Model): status: IntegerField for the status of the import session user: ForeignKey to the User who initiated the import field_defaults: JSONField for field default values - field_overrides: JSONField for field overrides """ @staticmethod @@ -83,7 +82,7 @@ def save(self, *args, **kwargs): ) user = models.ForeignKey( - User, on_delete=models.CASCADE, blank=True, null=True, verbose_name=_('User') + User, on_delete=models.SET_NULL, blank=True, null=True, verbose_name=_('User') ) field_defaults = models.JSONField( @@ -93,13 +92,6 @@ def save(self, *args, **kwargs): validators=[importer.validators.validate_field_defaults], ) - field_overrides = models.JSONField( - blank=True, - null=True, - verbose_name=_('Field Overrides'), - validators=[importer.validators.validate_field_defaults], - ) - @property def field_mapping(self): """Construct a dict of field mappings for this import session. @@ -188,18 +180,13 @@ def accept_mapping(self): # First, we need to ensure that all the *required* columns have been mapped required_fields = self.required_fields() - field_overrides = self.field_overrides or {} field_defaults = self.field_defaults or {} missing_fields = [] for field in required_fields.keys(): - # An override value exists - if field in field_overrides: - continue - # A default value exists - if field in field_defaults: + if field in field_defaults and field_defaults[field]: continue # The field has been mapped to a data column @@ -490,18 +477,12 @@ def extract_data( if not available_fields: available_fields = self.session.available_fields() - override_values = self.session.field_overrides or {} default_values = self.session.field_defaults or {} data = {} # We have mapped column (file) to field (serializer) already for field, col in field_mapping.items(): - # If an override value exists, use that - if field in override_values: - data[field] = override_values[field] - continue - # If this field is *not* mapped to any column, skip if not col: continue @@ -540,9 +521,6 @@ def serializer_data(self): if self.data: data.update(self.data) - if self.session.field_overrides: - data.update(self.session.field_overrides) - return data def construct_serializer(self): diff --git a/src/backend/InvenTree/importer/serializers.py b/src/backend/InvenTree/importer/serializers.py index 25ab1ab09217..61bcb269606f 100644 --- a/src/backend/InvenTree/importer/serializers.py +++ b/src/backend/InvenTree/importer/serializers.py @@ -47,7 +47,6 @@ class Meta: 'columns', 'column_mappings', 'field_defaults', - 'field_overrides', 'row_count', 'completed_row_count', ] From 10dd6cd43e506cea00fabcedd1724066f8854db9 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 22:41:48 +0000 Subject: [PATCH 188/190] Remove 'xls' as download option from frontend --- src/frontend/src/tables/DownloadAction.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/frontend/src/tables/DownloadAction.tsx b/src/frontend/src/tables/DownloadAction.tsx index 1de56b9803f7..e2d5c1d9ed25 100644 --- a/src/frontend/src/tables/DownloadAction.tsx +++ b/src/frontend/src/tables/DownloadAction.tsx @@ -21,7 +21,6 @@ export function DownloadAction({ const formatOptions = [ { value: 'csv', label: t`CSV`, icon: }, { value: 'tsv', label: t`TSV`, icon: }, - { value: 'xls', label: t`Excel (.xls)`, icon: }, { value: 'xlsx', label: t`Excel (.xlsx)`, icon: } ]; From 64b6460ecf73ac9156e5f9961abd66333b8bffda Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Fri, 5 Jul 2024 23:36:59 +0000 Subject: [PATCH 189/190] Add simple unit test for data import --- src/backend/InvenTree/importer/models.py | 2 +- .../importer/test_data/companies.csv | 13 ++++ src/backend/InvenTree/importer/tests.py | 63 ++++++++++++++++++- 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 src/backend/InvenTree/importer/test_data/companies.csv diff --git a/src/backend/InvenTree/importer/models.py b/src/backend/InvenTree/importer/models.py index f88615555347..3eb811c26239 100644 --- a/src/backend/InvenTree/importer/models.py +++ b/src/backend/InvenTree/importer/models.py @@ -254,8 +254,8 @@ def import_data(self): available_fields=available_fields, commit=False, ) - row.validate(commit=False) + row.valid = row.validate(commit=False) imported_rows.append(row) # Perform database writes as a single operation diff --git a/src/backend/InvenTree/importer/test_data/companies.csv b/src/backend/InvenTree/importer/test_data/companies.csv new file mode 100644 index 000000000000..8e5468b25b82 --- /dev/null +++ b/src/backend/InvenTree/importer/test_data/companies.csv @@ -0,0 +1,13 @@ +ID,Company name,Company description,Website,Phone number,Address,Email,Currency,Contact,Link,Image,Active,Is customer,Is manufacturer,Is supplier,Notes,Parts supplied,Parts manufactured,Address count +3,Arrow,Arrow Electronics,https://www.arrow.com/,,"70680 Shannon Rapid Apt. 570, 96124, Jenniferport, Arkansas, Holy See (Vatican City State)",,AUD,,,/media/company_images/company_3_img.jpg,True,False,False,True,,60,0,2 +1,DigiKey,DigiKey Electronics,https://www.digikey.com/,,"04964 Cox View Suite 815, 94832, Wesleyport, Delaware, Bolivia",,USD,,,/media/company_images/company_1_img.jpg,True,False,False,True,,200,0,2 +41,Future,Electronic components distributor,https://www.futureelectronics.com/,,"Wogan Terrace 79, 20157, Teasdale, Lebanon",,USD,,,/media/company_images/company_41_img.png,True,False,False,True,,60,0,4 +39,LCSC,Electronic components distributor,https://lcsc.com/,,"77673 Bishop Turnpike, 74969, North Cheryl, Hawaii, Portugal",,USD,,,/media/company_images/company_39_img.webp,True,False,False,True,,60,0,2 +38,McMaster-Carr,Supplier of mechanical components,https://www.mcmaster.com/,,"Schroeders Avenue 56, 8014, Sylvanite, Cayman Islands",,USD,,,/media/company_images/company_38_img.png,True,False,False,True,,240,0,1 +2,Mouser,Mouser Electronics,https://mouser.com/,,"Ashford Street 71, 24165, Leland, Jamaica",,AUD,,,/media/company_images/company_2_img.jpg,True,False,False,True,,61,0,2 +40,Newark,Online distributor of electronic components,https://www.newark.com/,,"Dekoven Court 3, 18301, Emison, Tuvalu",,USD,,,/media/company_images/company_40_img.png,True,False,False,True,,60,0,1 +36,Paint by Numbers,Supplier of high quality paint,,,"Orient Avenue 59, 18609, Corinne, Alabama, France, Metropolitan",,EUR,Pippy Painter,,/media/company_images/company_36_img.jpg,True,False,False,True,,15,0,1 +43,PCBWOY,PCB fabricator / supplier,,,"McKibben Street 77, 12370, Russellville, Benin",,USD,,,/media/company_images/company_43_img.png,True,False,False,True,,1,0,2 +29,Texas Instruments,,https://www.ti.com/,,"264 David Villages, 97718, Lake Michael, New Mexico, Kenya",,USD,,,/media/company_images/company_29_img.jpg,True,False,True,True,,0,1,2 +44,Wire-E-Coyote,American wire supplier,,,"Fountain Avenue 74, 12115, Gulf, Seychelles",,USD,,,,True,False,False,True,,5,0,3 +42,Wirey,Supplier of wire,,,"Preston Court 80, 4462, Manila, Russian Federation",,USD,,,/media/company_images/company_42_img.jpg,True,False,False,True,,11,0,2 diff --git a/src/backend/InvenTree/importer/tests.py b/src/backend/InvenTree/importer/tests.py index 638590551cf2..179d36dad990 100644 --- a/src/backend/InvenTree/importer/tests.py +++ b/src/backend/InvenTree/importer/tests.py @@ -1,5 +1,64 @@ """Unit tests for the 'importer' app.""" -from django.test import TestCase +import os -# Create your tests here. +from django.core.files.base import ContentFile + +from importer.models import DataImportSession +from InvenTree.unit_test import InvenTreeTestCase + + +class ImporterTest(InvenTreeTestCase): + """Basic tests for file imports.""" + + def test_import_session(self): + """Test creation of a data import session.""" + from company.models import Company + + n = Company.objects.count() + + fn = os.path.join(os.path.dirname(__file__), 'test_data', 'companies.csv') + + with open(fn, 'r') as input_file: + data = input_file.read() + + session = DataImportSession.objects.create( + data_file=ContentFile(data, 'companies.csv'), model_type='company' + ) + + session.extract_columns() + + self.assertEqual(session.column_mappings.count(), 14) + + # Check some of the field mappings + for field, col in [ + ('website', 'Website'), + ('is_customer', 'Is customer'), + ('phone', 'Phone number'), + ('description', 'Company description'), + ('active', 'Active'), + ]: + self.assertTrue( + session.column_mappings.filter(field=field, column=col).exists() + ) + + # Run the data import + session.import_data() + self.assertEqual(session.rows.count(), 12) + + # Check that some data has been imported + for row in session.rows.all(): + self.assertIsNotNone(row.data.get('name', None)) + self.assertTrue(row.valid) + + row.validate(commit=True) + self.assertTrue(row.complete) + + self.assertEqual(session.completed_row_count, 12) + + # Check that the new companies have been created + self.assertEqual(n + 12, Company.objects.count()) + + def test_field_defaults(self): + """Test default field values.""" + ... From c462eb713bc49883d2bf7ae338b9a01ae6644f38 Mon Sep 17 00:00:00 2001 From: Oliver Walters Date: Sat, 6 Jul 2024 07:46:49 +0000 Subject: [PATCH 190/190] PUI tweaks --- src/frontend/src/components/nav/NotificationDrawer.tsx | 5 ++++- src/frontend/src/components/render/StatusRenderer.tsx | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/frontend/src/components/nav/NotificationDrawer.tsx b/src/frontend/src/components/nav/NotificationDrawer.tsx index 6f72d3a79dbc..8d9a85483d07 100644 --- a/src/frontend/src/components/nav/NotificationDrawer.tsx +++ b/src/frontend/src/components/nav/NotificationDrawer.tsx @@ -21,6 +21,7 @@ import { Link, useNavigate } from 'react-router-dom'; import { api } from '../../App'; import { ApiEndpoints } from '../../enums/ApiEndpoints'; import { apiUrl } from '../../states/ApiState'; +import { useUserState } from '../../states/UserState'; import { StylishText } from '../items/StylishText'; /** @@ -33,10 +34,12 @@ export function NotificationDrawer({ opened: boolean; onClose: () => void; }) { + const { isLoggedIn } = useUserState(); + const navigate = useNavigate(); const notificationQuery = useQuery({ - enabled: opened, + enabled: opened && isLoggedIn(), queryKey: ['notifications', opened], queryFn: async () => api diff --git a/src/frontend/src/components/render/StatusRenderer.tsx b/src/frontend/src/components/render/StatusRenderer.tsx index 87e9cb89ba2a..42185690ee57 100644 --- a/src/frontend/src/components/render/StatusRenderer.tsx +++ b/src/frontend/src/components/render/StatusRenderer.tsx @@ -42,7 +42,9 @@ function renderStatusLabel( } if (!text) { - console.error(`renderStatusLabel could not find match for code ${key}`); + console.error( + `ERR: renderStatusLabel could not find match for code ${key}` + ); } // Fallbacks