Merge pull request #1318 from Signbank/manage_sentences_1227

Annotated Sentence List View

media/js/process_annotated_eaf_files.js

@@ -48,7 +48,7 @@ $(document).ready(function () {
     $('#annotatedSentenceForm').submit(function (event) {
         // Get all table rows except the header row
         var rows = $('#feedback table tbody tr');
-        var feedbackData = '';
+        var feedbackData = [];
 
         // Iterate over each table row
         rows.each(function() {
@@ -58,18 +58,23 @@ $(document).ready(function () {
             var starttime = row.find('td:eq(2)').text(); // Get the starttime value
             var endtime = row.find('td:eq(3)').text(); // Get the endtime value
 
-            // Concatenate gloss and its attributes with a separator
-            feedbackData += gloss + ':' + representative + ':' + starttime + ':' + endtime + ';';
+            // Concatenate gloss and its attributes to json format
+            feedbackData.push({
+                gloss: gloss,
+                representative: representative,
+                starttime: starttime,
+                endtime: endtime
+            });
         });
+
 
         // Append the feedback data to a hidden input field in the form
         $('<input>').attr({
             type: 'hidden',
             name: 'feedbackdata',
-            value: feedbackData
+            value: JSON.stringify(feedbackData)
         }).appendTo('#annotatedSentenceForm');
 
-        // Continue with form submission
         return true;
     });
 

signbank/dictionary/adminviews.py

@@ -71,14 +71,110 @@
                                                 senses_translations_per_language_list,
                                                 senses_sentences_per_language_list)
 from signbank.dictionary.context_data import (get_context_data_for_list_view, get_context_data_for_gloss_search_form,
-                                              get_web_search)
+                                                get_web_search)
 from signbank.dictionary.related_objects import (morpheme_is_related_to, gloss_is_related_to, gloss_related_objects,
                                                  okay_to_move_gloss, same_translation_languages, okay_to_move_glosses,
                                                  glosses_in_lemma_group, transitive_related_objects)
 from signbank.manage_videos import listing_uploaded_videos
 from signbank.zip_interface import *
 
 
+def order_annotatedsentence_queryset_by_sort_order(get, qs, queryset_language_codes):
+    """Change the sort-order of the query set, depending on the form field [sortOrder]
+
+    This function is used both by AnnotatedSentenceListView.
+    The value of [sortOrder] is 'firstgloss__id' by default.
+    [sortOrder] is a hidden field inside the "adminsearch" html form in the template admin_annotatedsentence_list.html
+    Its value is changed by clicking the up/down buttons in the second row of the search result table
+    """
+
+    def order_queryset_by_annotatedglosses(qs, sOrder):
+        print(sOrder)
+        # filter query on dataset and sort based on first gloss of the sentence
+        qs = qs.annotate(
+            firstgloss=Subquery(
+                AnnotatedGloss.objects.filter(
+                    annotatedsentence__id=OuterRef('id')
+                ).order_by('starttime').values('gloss__lemma')[:1]
+            )
+        ).order_by('firstgloss')
+
+        if sOrder[0:1] == '-':
+            # A starting '-' sign means: descending order
+            qs = qs.reverse()
+
+        return qs
+
+    def order_queryset_by_dataset(qs, sOrder):
+        qs = qs.annotate(
+            dataset=Subquery(
+                AnnotatedGloss.objects.filter(
+                    annotatedsentence_id=OuterRef('id')
+                ).values('gloss__lemma__dataset__acronym')[:1]
+            )
+        ).order_by('dataset')
+
+        if sOrder[0:1] == '-':
+            # A starting '-' sign means: descending order
+            qs = qs.reverse()
+        return qs
+
+    def order_queryset_by_sentencetranslation(qs, sOrder):
+        qs = qs.annotate(
+            translation=Subquery(
+                AnnotatedSentenceTranslation.objects.filter(
+                    annotatedsentence_id=OuterRef('id')
+                ).values('text')
+            )
+        ).order_by('translation')
+
+        if sOrder[0:1] == '-':
+            # A starting '-' sign means: descending order
+            qs = qs.reverse()
+        return qs
+
+    def order_queryset_by_nrannotatedglosses(qs, sOrder):
+        qs = qs.annotate(
+            nrglosses=Count('annotated_glosses')  # Replace 'glosses' with the related_name or field name
+        ).order_by('nrglosses')
+
+        if sOrder[0:1] == '-':
+            # A starting '-' sign means: descending order
+            qs = qs.reverse()
+        return qs
+
+    # Set the default sort order
+    bReversed = False
+    bText = True
+
+    # See if the form contains any sort-order information
+    if 'sortOrder' in get and get['sortOrder']:
+        # Take the user-indicated sort order
+        sOrder = get['sortOrder']
+        if sOrder[0:1] == '-':
+            # A starting '-' sign means: descending order
+            bReversed = True
+    else: 
+        return qs
+
+    # The ordering method depends on the kind of field:
+    # (1) text fields are ordered straightforwardly
+    # (2) fields made from a choice_list need special treatment
+    if sOrder.startswith("dataset_order_") or sOrder.startswith("-dataset_order_"):
+        ordered = order_queryset_by_dataset(qs, sOrder)
+    elif sOrder.startswith("annotatedsentencetranslation_order_") or sOrder.startswith("-annotatedsentencetranslation_order_"):
+        ordered = order_queryset_by_sentencetranslation(qs, sOrder)
+    elif sOrder.startswith("annotatedglossestranslation_order_") or sOrder.startswith("-annotatedglossestranslation_order_"):
+        ordered = order_queryset_by_annotatedglosses(qs, sOrder)
+    elif sOrder.startswith("nrannotatedglosses_order_") or sOrder.startswith("-nrannotatedglosses_order_"):
+        ordered = order_queryset_by_nrannotatedglosses(qs, sOrder)
+
+    if bReversed and bText:
+        ordered.reverse()
+
+    return ordered
+
+
 def order_queryset_by_sort_order(get, qs, queryset_language_codes):
     """Change the sort-order of the query set, depending on the form field [sortOrder]
 
@@ -241,6 +337,127 @@ def write(self, value):
         return value
 
 
+class AnnotatedSentenceListView(ListView):
+    model = AnnotatedSentence
+    template_name = 'dictionary/admin_annotatedsentence_list.html'
+    paginate_by = 10
+    show_all = False
+    search_type = 'annotatedsentence'
+    search_form = AnnotatedSentenceSearchForm()
+    queryset_language_codes = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def get_paginate_by(self, queryset):
+        return self.request.GET.get('paginate_by', self.paginate_by)
+
+    def get_queryset(self):
+        self.show_all = self.kwargs.get('show_all', False)
+
+        selected_datasets = get_selected_datasets_for_user(self.request.user)
+        dataset_languages = get_dataset_languages(selected_datasets)
+
+        get = self.request.GET
+        valid_regex, search_fields, field_values = check_language_fields_annotatedsentence(
+            self.search_form, AnnotatedSentenceSearchForm, get, dataset_languages
+        )
+
+        if USE_REGULAR_EXPRESSIONS and not valid_regex:
+            error_message_regular_expression(self.request, search_fields, field_values)
+            qs = AnnotatedSentence.objects.none()
+            return qs
+
+        if get.get('reset') == '1':
+            qs = AnnotatedSentence.objects.none()
+            return qs
+
+        # filter query on dataset and sort based on first gloss of the sentence
+        qs = AnnotatedSentence.objects.annotate(
+            dataset=Subquery(
+                AnnotatedGloss.objects.filter(
+                    annotatedsentence_id=OuterRef('id')
+                ).values('gloss__lemma__dataset')[:1]
+            ),
+            firstgloss=Subquery(
+                AnnotatedGloss.objects.filter(
+                    annotatedsentence__id=OuterRef('id')
+                ).order_by('starttime').values('gloss')[:1]
+            )
+        ).filter(dataset__in=selected_datasets).order_by('firstgloss')
+
+        default_dataset = Dataset.objects.get(acronym=settings.DEFAULT_DATASET_ACRONYM)
+
+        for lang in dataset_languages:
+            if lang.language_code_2char not in self.queryset_language_codes:
+                self.queryset_language_codes.append(lang.language_code_2char)
+        if not self.queryset_language_codes:
+            self.queryset_language_codes = [ default_dataset.default_language.language_code_2char ]
+        qs = order_annotatedsentence_queryset_by_sort_order(self.request.GET, qs, self.queryset_language_codes)
+
+        if get.get('show_all_annotatedsentences') == '1':
+            self.show_all = True
+
+        if get.get('no_glosses') == '1':
+            qs = qs.annotate(
+                num_annotated_glosses=Count('annotated_glosses')
+            ).filter(num_annotated_glosses=0)
+
+        if get.get('has_glosses') == '1':
+            qs = qs.annotate(
+                num_annotated_glosses=Count('annotated_glosses')
+            ).filter(num_annotated_glosses__gt=0)
+
+        if not self.show_all and not get:
+            qs = AnnotatedSentence.objects.none()
+
+        for get_key, get_value in get.items():
+            if get_key.startswith(AnnotatedSentenceSearchForm.annotatedsentence_search_field_prefix) and get_value:
+                language_code_2char = get_key[len(AnnotatedSentenceSearchForm.annotatedsentence_search_field_prefix):]
+                language = Language.objects.get(language_code_2char=language_code_2char)
+                qs = qs.filter(
+                    annotated_sentence_translations__text__icontains=get_value,
+                    annotated_sentence_translations__language=language
+                )
+
+        return qs
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+
+        set_up_language_fields(AnnotatedSentence, self, self.search_form)
+        selected_datasets = get_selected_datasets_for_user(self.request.user)
+        dataset_languages = get_dataset_languages(selected_datasets)
+        nr_sentences_in_dataset = AnnotatedSentence.objects.annotate(
+            dataset=Subquery(
+                AnnotatedGloss.objects.filter(
+                    annotatedsentence_id=OuterRef('id')
+                ).values('gloss__lemma__dataset')[:1]
+            )
+        ).filter(dataset__in=selected_datasets).count()
+        results = self.get_queryset()
+
+        get = self.request.GET
+        query_parameters_dict = {}
+        query_parameters_keys = []
+        for key in list(get.keys()):
+            query_parameters_dict[key] = get.get(key)
+            query_parameters_keys.append(key)
+        context['sort_order'] = str(get.get('sortOrder'))
+        context['language_query_keys'] = [language.language_code_2char for language in dataset_languages]
+        context['query_parameters_dict'] = query_parameters_dict
+        context['query_parameters_keys'] = query_parameters_keys
+        context['annotatedsentence_count'] = nr_sentences_in_dataset
+        context['dataset_languages'] = dataset_languages
+        context['USE_REGULAR_EXPRESSIONS'] = getattr(settings, 'USE_REGULAR_EXPRESSIONS', False)
+        context['searchform'] = self.search_form
+        context['show_all'] = self.show_all
+        context['search_type'] = self.search_type
+        context['search_matches'] = results.count()
+
+        return context
+
+
 class GlossListView(ListView):
 
     model = Gloss

signbank/dictionary/forms.py

@@ -8,7 +8,7 @@
 from signbank.video.fields import VideoUploadToFLVField
 from signbank.dictionary.models import (Dialect, Gloss, Morpheme, Definition, Relation, RelationToForeignSign,
                                         MorphologyDefinition, OtherMedia, Handshape, SemanticField, DerivationHistory,
-                                        AnnotationIdglossTranslation, Dataset, FieldChoice, LemmaIdgloss,
+                                        AnnotationIdglossTranslation, Dataset, FieldChoice, LemmaIdgloss, AnnotatedSentence,
                                         LemmaIdglossTranslation, Translation, Keyword, Language, SignLanguage,
                                         QueryParameterFieldChoice, SearchHistory, QueryParameter,
                                         QueryParameterMultilingual, QueryParameterHandshape, SemanticFieldTranslation,
@@ -415,6 +415,42 @@ def check_language_fields(searchform, formclass, queryDict, languages):
 
     return language_fields_okay, search_fields, field_values
 
+def check_language_fields_annotatedsentence(searchform, formclass, queryDict, languages):
+    # this function inspects the search parameters from GlossSearchForm looking for occurrences of + at the start
+    language_fields_okay = True
+    search_fields = []
+    field_values = []
+    if not queryDict or not USE_REGULAR_EXPRESSIONS:
+        return language_fields_okay, search_fields, field_values
+
+    language_field_labels = dict()
+    language_field_values = dict()
+    for language in languages:
+        # add for AnnotatedSentence
+        if hasattr(searchform, 'annotatedsentence_search_field_prefix'):
+            annotatedsentence_field_name = formclass.annotatedsentence_search_field_prefix + language.language_code_2char
+            if annotatedsentence_field_name in queryDict.keys():
+                language_field_values[annotatedsentence_field_name] = queryDict[annotatedsentence_field_name]
+
+    import re
+    # check for matches starting with: + * [ ( ) ?
+    # or ending with a +
+    regexp = re.compile(r'^[+*\[()?]|([^+]+\+$)')
+    for language_field in language_field_values.keys():
+        try:
+            re.compile(language_field_values[language_field])
+        except re.error:
+            language_fields_okay = False
+            search_fields.append(language_field_labels[language_field])
+            field_values.append(language_field_values[language_field])
+            break
+        if regexp.search(language_field_values[language_field]):
+            language_fields_okay = False
+            search_fields.append(language_field_labels[language_field])
+            field_values.append(language_field_values[language_field])
+
+    return language_fields_okay, search_fields, field_values
+
 
 class MorphemeSearchForm(forms.ModelForm):
     use_required_attribute = False  # otherwise the html required attribute will show up on every form
@@ -822,6 +858,27 @@ class ImageUploadForHandshapeForm(forms.Form):
     redirect = forms.CharField(widget=forms.HiddenInput, required=False)
 
 
+class AnnotatedSentenceSearchForm(forms.ModelForm):
+    use_required_attribute = False  # otherwise the html required attribute will show up on every form
+
+    search = forms.CharField(label=_("Search"))
+    sortOrder = forms.CharField(label=_("Sort Order"))
+    no_glosses = forms.ChoiceField(label=_('Only show results without glosses'), choices=[],
+                                   widget=forms.Select(attrs=ATTRS_FOR_BOOLEAN_FORMS))
+    has_glosses = forms.ChoiceField(label=_('Only show results with glosses'), choices=[],
+                                    widget=forms.Select(attrs=ATTRS_FOR_BOOLEAN_FORMS))
+    annotatedsentence_search_field_prefix = "annotatedsentence_"
+
+    class Meta:
+        model = AnnotatedSentence
+        fields = ['search']
+
+    def __init__(self, *args, **kwargs):
+        super(AnnotatedSentenceSearchForm, self).__init__(*args, **kwargs)
+
+        for boolean_field in ['no_glosses', 'has_glosses']:
+            self.fields[boolean_field].choices = [(0, _('No')), (1, _('Yes'))]
+
 class LemmaSearchForm(forms.ModelForm):
     use_required_attribute = False  # otherwise the html required attribute will show up on every form