Skip to content

Commit

Permalink
Add missing grade_cells before autograding
Browse files Browse the repository at this point in the history
  • Loading branch information
tuncbkose committed Aug 21, 2023
1 parent a81aa54 commit e4d8b4f
Show file tree
Hide file tree
Showing 2 changed files with 194 additions and 3 deletions.
141 changes: 141 additions & 0 deletions nbgrader/preprocessors/overwritecells.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,84 @@
from .. import utils
from ..api import Gradebook, MissingEntry
from . import NbGraderPreprocessor
from ..nbgraderformat import MetadataValidator
from nbconvert.exporters.exporter import ResourcesDict
from nbformat.notebooknode import NotebookNode
from traitlets import Bool, Unicode
from typing import Tuple, Any
from textwrap import dedent


class OverwriteCells(NbGraderPreprocessor):
"""A preprocessor to overwrite information about grade and solution cells."""

add_missing_cells = Bool(
False,
help=dedent(
"""
Whether or not missing grade_cells should be added back
to the notebooks being graded.
"""
),
).tag(config=True)

missing_cell_notification = Unicode(
"This cell (id:{cell_id}) was missing from the submission. " +
"It was added back by nbgrader.\n\n", # Markdown requires two newlines
help=dedent(
"""
A text to add at the beginning of every missing cell re-added to the notebook during autograding.
"""
)
).tag(config=True)

def missing_cell_transform(self, source_cell, max_score, is_solution=False, is_task=False):
"""
Converts source_cell obtained from Gradebook into a cell that can be added to the notebook.
It is assumed that the cell is a grade_cell (unless is_task=True)
"""

missing_cell_notification = self.missing_cell_notification.format(cell_id=source_cell.name)

cell = {
"cell_type": source_cell.cell_type,
"metadata": {
"deletable": False,
"editable": False,
"nbgrader": {
"grade": True,
"grade_id": source_cell.name,
"locked": source_cell.locked,
"checksum": source_cell.checksum,
"cell_type": source_cell.cell_type,
"points": max_score,
"solution": False
}
},
"source": missing_cell_notification + source_cell.source
}

# Code cell format is slightly different
if cell["cell_type"] == "code":
cell["execution_count"] = None
cell["outputs"] = []
cell["source"] = "# " + cell["source"] # make the notification we add a comment

# some grade cells are editable (manually graded answers)
if is_solution:
del cell["metadata"]["editable"]
cell["metadata"]["nbgrader"]["solution"] = True
# task cells are also a bit different
elif is_task:
cell["metadata"]["nbgrader"]["grade"] = False
cell["metadata"]["nbgrader"]["task"] = True
# this is when task cells were added so metadata validation should start from here
cell["metadata"]["nbgrader"]["schema_version"] = 3

cell = NotebookNode(cell)
cell = MetadataValidator().upgrade_cell_metadata(cell)
return cell

def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
# pull information from the resources
self.notebook_id = resources['nbgrader']['notebook']
Expand All @@ -22,6 +92,77 @@ def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[Notebo

with self.gradebook:
nb, resources = super(OverwriteCells, self).preprocess(nb, resources)
if self.add_missing_cells:
nb, resources = self.add_missing_grade_cells(nb, resources)
nb, resources = self.add_missing_task_cells(nb, resources)

return nb, resources

def add_missing_grade_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
"""
Add missing grade cells back to the notebook.
If missing, find the previous solution/grade cell, and add the current cell after it.
It is assumed such a cell exists because
presumably the grade_cell exists to grade some work in the solution cell.
"""
source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id)
source_cells = source_nb.source_cells
source_cell_ids = [cell.name for cell in source_cells]
grade_cells = {cell.name: cell for cell in source_nb.grade_cells}
solution_cell_ids = [cell.name for cell in source_nb.solution_cells]

# track indices of solution and grade cells in the submitted notebook
submitted_cell_idxs = dict()
for idx, cell in enumerate(nb.cells):
if utils.is_grade(cell) or utils.is_solution(cell):
submitted_cell_idxs[cell.metadata.nbgrader["grade_id"]] = idx

# Every time we add a cell, the idxs above get shifted
# We could process the notebook backwards, but that makes adding the cells in the right place more difficult
# So we keep track of how many we have added so far
added_count = 0

for grade_cell_id, grade_cell in grade_cells.items():
# If missing, find the previous solution/grade cell, and add the current cell after it.
if grade_cell_id not in submitted_cell_idxs:
self.log.warning(f"Missing grade cell {grade_cell_id} encountered, adding to notebook")
source_cell_idx = source_cell_ids.index(grade_cell_id)
cell_to_add = source_cells[source_cell_idx]
cell_to_add = self.missing_cell_transform(cell_to_add, grade_cell.max_score,
is_solution=grade_cell_id in solution_cell_ids)
# First cell was deleted, add it to start
if source_cell_idx == 0:
nb.cells.insert(0, cell_to_add)
submitted_cell_idxs[grade_cell_id] = 0
# Deleted cell is not the first, add it after the previous solution/grade cell
else:
prev_cell_id = source_cell_ids[source_cell_idx - 1]
prev_cell_idx = submitted_cell_idxs[prev_cell_id] + added_count
nb.cells.insert(prev_cell_idx + 1, cell_to_add) # +1 to add it after
submitted_cell_idxs[grade_cell_id] = submitted_cell_idxs[prev_cell_id]

# If the cell we just added is followed by other missing cells, we need to know its index in the nb
# However, no need to add `added_count` to avoid double-counting

added_count += 1 # shift idxs

return nb, resources

def add_missing_task_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
"""
Add missing task cells back to the notebook.
We can't figure out their original location, so they are added at the end, in their original order.
"""
source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id)
source_cells = source_nb.source_cells
source_cell_ids = [cell.name for cell in source_cells]
submitted_ids = [cell["metadata"]["nbgrader"]["grade_id"] for cell in nb.cells if
"nbgrader" in cell["metadata"]]
for task_cell in source_nb.task_cells:
if task_cell.name not in submitted_ids:
cell_to_add = source_cells[source_cell_ids.index(task_cell.name)]
cell_to_add = self.missing_cell_transform(cell_to_add, task_cell.max_score, is_task=True)
nb.cells.append(cell_to_add)

return nb, resources

Expand Down
56 changes: 53 additions & 3 deletions nbgrader/tests/preprocessors/test_overwritecells.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import pytest

from nbformat.v4 import new_notebook
from nbformat.v4 import new_notebook, new_markdown_cell

from ...preprocessors import SaveCells, OverwriteCells
from ...api import Gradebook
from ...utils import compute_checksum
from .base import BaseTestPreprocessor
from .. import (
create_grade_cell, create_solution_cell, create_grade_and_solution_cell,
create_locked_cell)
create_locked_cell, create_task_cell)


@pytest.fixture
Expand All @@ -23,6 +23,7 @@ def gradebook(request, db):

def fin():
gb.close()

request.addfinalizer(fin)

return gb
Expand Down Expand Up @@ -197,7 +198,7 @@ def test_overwrite_locked_checksum(self, preprocessors, resources):

assert cell.metadata.nbgrader["checksum"] == compute_checksum(cell)

def test_nonexistant_grade_id(self, preprocessors, resources):
def test_nonexistent_grade_id(self, preprocessors, resources):
"""Are cells not in the database ignored?"""
cell = create_grade_cell("", "code", "", 1)
cell.metadata.nbgrader['grade'] = False
Expand All @@ -215,3 +216,52 @@ def test_nonexistant_grade_id(self, preprocessors, resources):
nb, resources = preprocessors[1].preprocess(nb, resources)
assert 'grade_id' not in cell.metadata.nbgrader

# Tests for adding missing cells back
def test_add_missing_cells(self, preprocessors, resources):
"""
Note: This test will produce warnings (from OverwriteCells preprocessor) by default.
Current implementation of adding missing cells should:
- add missing cells right after the previous grade/solution cell, as the best approximation of their location
- add task cells at the end (because we can't detect their location in the notebook), in order of appearance
"""

cells = [
create_solution_cell("Code assignment", "code", "code_solution"),
create_grade_cell("some tests", "code", "code_test1", 1),
create_grade_cell("more tests", "code", "code_test2", 1),
new_markdown_cell("some description"),
create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded1", 1),
create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded2", 1),
new_markdown_cell("some description"),
create_task_cell("some task description", "markdown", "task_cell1", 1),
new_markdown_cell("some description"),
create_task_cell("some task description", "markdown", "task_cell2", 1),
]
# Add checksums to suppress warning
nbgrader_cells = [0, 1, 2, 4, 5, 7, 9]
for idx, cell in enumerate(cells):
if idx in nbgrader_cells:
cell.metadata.nbgrader["checksum"] = compute_checksum(cell)

expected_order = [0, 1, 2, 4, 5, 3, 6, 8, 7, 9]
expected = [cells[i].metadata.nbgrader["grade_id"] if "nbgrader" in cells[i].metadata else "markdown" for i in expected_order]

nb = new_notebook()
nb.cells = cells

# save to database
nb, resources = preprocessors[0].preprocess(nb, resources)

# remove grade/task cells to test their restoration
nb.cells.pop(9)
nb.cells.pop(7)
nb.cells.pop(5)
nb.cells.pop(4)
nb.cells.pop(2)
nb.cells.pop(1)

# restore
preprocessors[1].add_missing_cells = True
nb, resources = preprocessors[1].preprocess(nb, resources)
result = [cell["metadata"]["nbgrader"]["grade_id"] if "nbgrader" in cell["metadata"] else "markdown" for cell in nb.cells]
assert expected == result

0 comments on commit e4d8b4f

Please sign in to comment.