Publication Date Parsing

requirements.txt

@@ -1,6 +1,7 @@
 asgiref>=3.3.0
 celery==5.0.5
 cssselect==1.1.0
+datefinder==0.7.3
 Django==3.2.16
 django-elasticsearch-dsl==7.2.2
 django-filter==2.4.0

swp/admin/publication.py

@@ -15,6 +15,7 @@ class PublicationAdmin(admin.ModelAdmin):
         'authors',
         'abstract',
         'publication_date',
+        'publication_dt',
         'last_access',
         'url',
         'pdf_url',

swp/api/serializers/publication.py

@@ -24,6 +24,7 @@ class Meta:
             'authors',
             'abstract',
             'publication_date',
+            'publication_dt',
             'last_access',
             'url',
             'pdf_url',

swp/documents/publication.py

@@ -103,6 +103,7 @@ class Django:
             'ris_type',
             'authors',
             'publication_date',
+            'publication_dt',
             'last_access',
             'url',
             'pdf_url',

swp/forms/publication.py

@@ -12,6 +12,8 @@
 from django.utils.translation import gettext_lazy as _
 
 from swp.models import Publication
+from swp.utils.date import parse_date
+
 if TYPE_CHECKING:
     from swp.models import Thinktank
 
@@ -64,6 +66,11 @@ def clean_authors(self) -> Sequence[str]:
 
         return [clean(author) for author in items]
 
+    def clean_publication_dt(self) -> datetime.date:
+        value = self.truncated_field('publication_date')
+        default = f'{datetime.date.today().year}-01-01'
+        return parse_date(value, default_date_str=default)
+
     def clean(self) -> Mapping[str, Any]:
         super().clean()
 

swp/migrations/0040_publication_publication_dt.py

@@ -0,0 +1,30 @@
+# Generated by Django 3.2.16 on 2023-04-13 14:56
+
+from django.db import migrations, models
+
+from swp.utils.date import parse_date
+
+
+def migrate_date(apps, schema_editor):
+    Publication = apps.get_model('swp', 'Publication')
+
+    for publication in Publication.objects.all():
+        default = publication.created.date().isoformat()
+        publication.publication_dt = parse_date(publication.publication_date, default_date_str=default)
+        publication.save(update_fields=['publication_dt'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('swp', '0039_publication_list'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='publication',
+            name='publication_dt',
+            field=models.DateField(blank=True, null=True, verbose_name='publication dt'),
+        ),
+        migrations.RunPython(code=migrate_date),
+    ]

swp/models/publication.py

@@ -42,6 +42,7 @@ class Publication(models.Model):
     abstract = models.TextField(_('abstract'), blank=True)  # [AB]
     authors = ArrayField(models.CharField(max_length=255), blank=True, null=True, verbose_name=_('authors'))  # [AU]
     publication_date = models.CharField(_('publication date'), max_length=255, blank=True, default='')  # [PY]
+    publication_dt = models.DateField(_('publication dt'), null=True, blank=True)
     last_access = models.DateTimeField(_('last access'), default=timezone.now, editable=False)  # [Y2]
     url = LongURLField(_('URL'))  # [UR]
     pdf_url = LongURLField(_('PDF URL'), blank=True)  # [L1]

swp/utils/date.py

@@ -0,0 +1,28 @@
+import re
+import datetime as dt
+
+import datefinder
+
+
+def parse_date(date_str: str, default_date_str: str='1700-01-01') -> dt.date:
+    date_str = ' '.join(date_str.split())
+    default_date = dt.datetime.fromisoformat(default_date_str).date()
+    if not date_str:
+        return default_date
+    try:
+        return dt.datetime.fromisoformat(date_str).date()
+    except ValueError:
+        pass
+    year = default_date.year
+    years_found = re.findall(r'(19\d{2}|20\d{2})', date_str)
+    if years_found:
+        year = int(sorted(years_found)[0])
+    if year != default_date.year:
+        default_date = dt.date(year, 1, 1)
+    dates_found = sorted([d.date() for d in datefinder.find_dates(date_str)])
+    date = dates_found[0] if dates_found else None
+    if date is None:
+        return default_date
+    if years_found and date.year != default_date.year:
+        return default_date
+    return date