add script to fetch new professors/courses

api/serializers.py

@@ -48,17 +48,8 @@ def to_representation(self, data):
 
 class ProfessorListSerializer(ListSerializer):
     def to_representation(self, data):
-        # one would expect to be able to write
-        # `data.filter(status=Professor.Status.VERIFIED)` here, but filtering an
-        # already-sliced queryset is prohibited by django due to ambiguity
-        # concerns: https://docs.djangoproject.com/en/3.2/ref/models/querysets/.
-        # I can't see how it would be ambiguous unless you did something like
-        # slice-filter-slice, but oh well.
-        # We should be careful in the future that this doesn't cause performance
-        # issues; this relies on slicing already having occured before we
-        # receive `data`, limiting it to a reasonable 100 professors.
-        data = [p for p in data if p.status == Professor.Status.VERIFIED]
-        return super().to_representation(data)
+        data = data.filter(status=Professor.Status.VERIFIED)
+        return super().to_representation()
 
 class ReviewsSerializer(ModelSerializer):
     course = CourseField()

home/forms/admin_forms.py

@@ -531,10 +531,10 @@ def clean(self):
 # Used on /admin when verifying professors that might be duplicates of already
 # verified professors.
 class ProfessorInfoModal(Form):
-    def __init__(self, unverified_professor: Professor, verified_professor: Professor):
+    def __init__(self, unverified_professor, verified_professors):
         super().__init__()
         self.unverified_professor = unverified_professor
-        self.verified_professor = verified_professor
+        self.verified_professors = verified_professors
         self.helper = FormHelper()
         self.helper.form_tag = False
         self.helper.layout = self.generate_layout()
@@ -557,44 +557,46 @@ def get_courses(professor: Professor):
 
             return "No Courses" if len(courses) == 0 else ', '.join(course.name for course in courses)
 
+        def create_row(professor: Professor):
+            row = '''
+                <tr>
+                    <td><a href="{absolute_url}" target="_blank">{verified_name}</a></td>
+                    <td>{verified_courses}</td>
+                    <td><button class="btn btn-primary" onclick="mergeProfessor({merge_args})">Merge</button></td>
+                </tr>
+            '''
+
+            merge_data = {
+                "merge_subject": self.unverified_professor.name,
+                "subject_id": self.unverified_professor.pk,
+                "merge_target": professor.name,
+                "target_id": professor.pk
+            }
+
+            kwargs = {
+                "absolute_url": professor.get_absolute_url(),
+                "verified_name": professor.name,
+                "verified_courses": get_courses(professor),
+                "merge_args": merge_data
+            }
+            return format_html(row, **kwargs)
+
         table_str = '''
-            <table class="table">
-                <thead>
-                    <tr>
-                        <th>Name</th>
-                        <th>Courses</th>
-                    </tr>
-                </thead>
+            <table class="table text-center w-100">
                 <tbody>
-                    <tr>
-                        <td>{unverified_name} (this professor)</td>
-                        <td>{unverified_courses}</td>
-                    </tr>
-                    <tr>
-                        <td><a href="{absolute_url}" target="_blank">{verified_name}</a></td>
-                        <td>{verified_courses}</td>
-                    </tr>
+        '''
+        for professor in self.verified_professors:
+            table_str += create_row(professor)
+
+        table_str += '''
                 </tbody>
             </table>
         '''
-        kwargs = {
-            "unverified_name": self.unverified_professor.name,
-            "unverified_courses": get_courses(self.unverified_professor),
-            "absolute_url": self.verified_professor.get_absolute_url(),
-            "verified_name": self.verified_professor.name,
-            "verified_courses": get_courses(self.verified_professor)
-        }
 
         modal_title = (
-            f'This {self.unverified_professor.type} might be a duplicate of <b>{self.verified_professor.name} ({self.verified_professor.pk})</b>. <br>'
-            f'Given the information below, please decide if these {self.unverified_professor.type}s are the same.'
+            f'This {self.unverified_professor.type} might be a duplicate of one of the professors below. <br>'
+            f'{self.unverified_professor.name} has taught: {get_courses(self.unverified_professor)}'
         )
-        merge_data = {
-            "merge_subject": self.unverified_professor.name,
-            "subject_id": self.unverified_professor.pk,
-            "merge_target": self.verified_professor.name,
-            "target_id": self.verified_professor.pk
-        }
 
         verify_data = {
             "professor_id": self.unverified_professor.pk,
@@ -604,14 +606,10 @@ def get_courses(professor: Professor):
 
         return Layout(
             Modal(
-                HTML(format_html(table_str, **kwargs)),
-                Div(
-                    Button("verify", "Verify", css_class="btn btn-success", onclick=format_html("verifyProfessor({args})", args=verify_data)),
-                    Button("merge", "Merge", css_class="btn btn-primary", onclick=format_html("mergeProfessor({args})", args=merge_data)),
-                    css_class="btn-group w-100"
-                ),
+                HTML(table_str),
+                Button("verify", "Verify", css_class="btn btn-success w-100", onclick=format_html("verifyProfessor({args})", args=verify_data)),
                 css_id="info-modal",
                 title=format_html(modal_title),
-                title_class="text-center"
+                title_class="text-center w-100"
             )
         )

home/forms/professor_forms.py

@@ -205,7 +205,7 @@ class ProfessorFormReview(ProfessorForm):
     def __init__(self, user, professor, **kwargs):
         super().__init__(user, Review.ReviewType.REVIEW, **kwargs)
 
-        courses = professor.course_set.order_by("name").distinct()
+        courses = professor.course_set.order_by("name")
         choices = [(course.name, course.name) for course in courses]
         choices.insert(0, ('', "Course"))
         choices.append(("other", "Other"))

home/management/commands/updatecourses.py

@@ -0,0 +1,95 @@
+import requests
+from datetime import datetime
+
+from django.core.management import BaseCommand
+
+from home.models import Course, Professor, ProfessorCourse
+from home.utils import Semester
+
+class Command(BaseCommand):
+    def __init__(self):
+        super().__init__()
+        self.total_num_new_courses = 0
+        self.total_num_new_professors = 0
+
+    def add_arguments(self, parser):
+        parser.add_argument("semesters", nargs='+')
+
+    def handle(self, *args, **options):
+        t_start = datetime.now()
+        semesters = [Semester(s) for s in options['semesters']]
+        print(f"Inputted Semesters: {', '.join(s.name() for s in semesters)}")
+
+        for semester in semesters:
+            kwargs = {"semester": semester, "per_page": 100, "page": 1}
+            course_data = requests.get("https://api.umd.io/v1/courses", params=kwargs).json()
+
+            if "error_code" in course_data[0].keys():
+                print(f"umd.io doesn't have data for {semester.name()}!")
+                continue
+
+            print(f"Working on courses for {semester.name()}...")
+
+            while course_data:
+                for umdio_course in course_data:
+                    course = Course.unfiltered.filter(name=umdio_course['course_id']).first()
+                    if not course:
+                        course = Course(
+                            name=umdio_course['course_id'],
+                            department=umdio_course['dept_id'],
+                            course_number=umdio_course['course_id'][4:],
+                            title=umdio_course['name'],
+                            credits=umdio_course['credits'],
+                            description=umdio_course["description"]
+                        )
+
+                        course.save()
+                        self.total_num_new_courses += 1
+
+                    self._professors(course, semester)
+                    print(course)
+
+                kwargs["page"] += 1
+                course_data = requests.get("https://api.umd.io/v1/courses", params=kwargs).json()
+
+        print(f"\n** New Courses Created: {self.total_num_new_courses} **")
+        print(f"** New Professors Created: {self.total_num_new_professors} **")
+
+        runtime = datetime.now() - t_start
+        print(f"Runtime: {round(runtime.seconds / 60, 2)} minutes")
+
+    def _professors(self, course: Course, semester: Semester):
+        kwargs = {"course_id": course.name}
+        umdio_professors = requests.get("https://api.umd.io/v1/professors", params=kwargs).json()
+
+        # if no professors were found for `course` during `semester`
+        if isinstance(umdio_professors, dict) and 'error_code' in umdio_professors.keys():
+            return
+
+        for umdio_professor in umdio_professors:
+            if umdio_professor['name'] == "Instructor: TBA":
+                continue
+
+            professor = Professor.unfiltered.filter(name=umdio_professor['name']).first()
+
+            if not professor:
+                # To make our lives easier, attempt to automatically verify the professor
+                # following the same criteria in admin.py
+                split_name = umdio_professor['name'].strip().split()
+                similar_professors = Professor.find_similar(professor.name, 70)
+                split_name = str(professor.name).strip().split(" ")
+                new_slug = "_".join(reversed(split_name)).lower()
+
+                professor = Professor(name=umdio_professor['name'], type=Professor.Type.PROFESSOR)
+
+                if len(similar_professors) == 0 and not Professor.verified.filter(slug=new_slug).exists():
+                    professor.slug = "_".join(reversed(split_name)).lower()
+                    professor.status = Professor.Status.VERIFIED
+
+                professor.save()
+                self.total_num_new_professors += 1
+
+            for entry in umdio_professor['taught']:
+                if entry['course_id'] == course.name and Semester(entry['semester']) == semester:
+                    ProfessorCourse.objects.create(course=course, professor=professor, recent_semester=semester)
+                    break

home/models.py

@@ -11,6 +11,8 @@
     CASCADE, ManyToManyField, SlugField, TextChoices, FloatField, Manager,
     QuerySet, Sum, UniqueConstraint, Index, Count)
 
+from fuzzywuzzy import fuzz
+
 class GradeQuerySet(QuerySet):
 
     def exclude_pf(self):
@@ -264,10 +266,24 @@ def average_rating(self):
     def get_absolute_url(self):
         return reverse("professor", kwargs={"slug": self.slug})
 
+    @staticmethod
+    def find_similar(professor_name, tolerance):
+        similar_professors = []
+        for professor in Professor.verified.all():
+            if fuzz.ratio(professor_name, professor.name) > tolerance:
+                similar_professors.append(professor)
+            ratio = fuzz.ratio(professor_name, professor.name)
+            if ratio > tolerance:
+                similar_professors.append({"professor" : professor, "ratio" : ratio})
+
+        similar_professors.sort(key=lambda e: e["ratio"], reverse=True)
+        similar_professors = [p["professor"] for p in similar_professors]
+
+        return similar_professors
+
     def __str__(self):
         return f"{self.name} ({self.id})"
 
-
 class ProfessorAlias(Model):
     class Meta:
         db_table = "home_professor_alias"

home/utils.py

@@ -69,20 +69,14 @@ def from_name(cls, name):
     @staticmethod
     def current():
         now = datetime.now()
-        # spring (current year)
-        if now.month < 3:
-            semester = "01"
-            year = now.year
         # fall
         if 3 <= now.month <= 9:
             semester = "08"
-            year = now.year
-        # spring (of next year)
-        if 9 < now.month:
+        # spring
+        else:
             semester = "01"
-            year = now.year + 1
 
-        return Semester(f"{year}{semester}")
+        return Semester(f"{now.year}{semester}")
 
     def name(self, *, year_first=False, short=False):
         year = self.year

home/views/admin.py

@@ -3,16 +3,16 @@
 
 from django.shortcuts import render
 from django.views import View
-from django.db.models import Q
 from django.http import JsonResponse
 from django.template.context_processors import csrf
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.urls import reverse
+from django.utils.safestring import mark_safe
 
 from crispy_forms.utils import render_crispy_form
 from discord_webhook import DiscordWebhook, DiscordEmbed
 
-from home.models import Review, Professor, ProfessorAlias, ProfessorCourse, Grade, User
+from home.models import Review, Professor, ProfessorCourse, ProfessorAlias, Grade, User
 from home.utils import AdminAction
 from home.tables.reviews_table import UnverifiedReviewsTable
 from home.tables.basic import ProfessorsTable
@@ -86,7 +86,6 @@ def post(self, request):
                 username = "Anonymous" if (review.anonymous or not review.user) else review.user.username
 
                 embed.add_embed_field(name="Reviewer", value=username, inline=True)
-                embed.add_embed_field(name="Rating", value=review.rating, inline=True)
                 embed.add_embed_field(name="Course", value=course, inline=True)
                 embed.add_embed_field(name="Grade", value=grade, inline=True)
                 embed.add_embed_field(name="Review", value=review_text, inline=False)
@@ -167,7 +166,7 @@ def post(self, request):
                 ProfessorCourse.objects.filter(professor__id=subject_id).update(professor=merge_target)
                 Review.unfiltered.filter(professor__id=subject_id).update(professor=merge_target)
                 Grade.unfiltered.filter(professor__id=subject_id).update(professor=merge_target)
-                
+
                 aliases = ProfessorAlias.objects.filter(name=merge_subject.name, professor=merge_target)
                 if not aliases.exists():
                     ProfessorAlias(name=merge_subject.name, professor=merge_target).save()
@@ -299,34 +298,32 @@ def verify_professor(self, verified_status: Professor.Status, slug: Optional[str
                 response["error_msg"] = self.not_found_err("Professor")
                 return JsonResponse(response)
             if not professor.slug and slug is None:
-                # Attempt to create slug automatically
-                split_name = str(professor.name).strip().split(" ")
-                first_name = split_name[0].lower().strip()
-                last_name = split_name[-1].lower().strip()
-
-                query = Professor.verified.filter(
-                    (
-                        Q(name__istartswith=first_name) &
-                        Q(name__iendswith=last_name)
-                    ) |
-                    Q(slug="_".join(reversed(split_name)).lower())
-                )
                 ctx = {}
                 ctx.update(csrf(request))
 
+                similar_professors = Professor.find_similar(professor.name)
                 verify_override = json.loads(request.POST["override"])
-                if not verify_override and query.exists():
-                    form = ProfessorInfoModal(professor, query[0])
+
+                if not verify_override and len(similar_professors) > 0:
+                    form = ProfessorInfoModal(professor, similar_professors)
                     response["form"] = render_crispy_form(form, form.helper, context=ctx)
                     response["success_msg"] = "#info-modal-container"
                     return JsonResponse(response)
 
+                split_name = str(professor.name).strip().split(" ")
+                new_slug = "_".join(reversed(split_name)).lower()
+                modal_msg = None
+
+                if Professor.verified.filter(slug=new_slug).exists():
+                    modal_msg = mark_safe(f"The slug <b>{new_slug}</b> already belongs to a professor. Please enter a slug below.")
+
                 if len(split_name) > 2:
                     modal_msg = (
                         f"The name '{professor.name}' is too long and "
                         "can't be slugged automatcially. Please enter a slug below."
                     )
 
+                if modal_msg:
                     # Create the modal form to manualy enter a slug and add it
                     # to the response. The form creates the modal, though it's
                     # actually summoned from admin-action.js
@@ -335,7 +332,7 @@ def verify_professor(self, verified_status: Professor.Status, slug: Optional[str
                     response["success_msg"] = "#slug-modal-container"
                     return JsonResponse(response)
 
-                professor.slug = "_".join(reversed(split_name)).lower()
+                professor.slug = new_slug
         else:
             professor.slug = None
             if verified_status is Professor.Status.REJECTED:

home/views/professor.py

@@ -33,7 +33,6 @@ def get(self, request, slug):
             Course.recent
             .filter(professors__pk=professor.pk)
             .order_by("name")
-            .distinct()
         )
 
         courses_reviewed = []