Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add missing grade_cells before autograding #1770

Merged
merged 1 commit into from
Aug 21, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 141 additions & 0 deletions nbgrader/preprocessors/overwritecells.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,84 @@
from .. import utils
from ..api import Gradebook, MissingEntry
from . import NbGraderPreprocessor
from ..nbgraderformat import MetadataValidator
from nbconvert.exporters.exporter import ResourcesDict
from nbformat.notebooknode import NotebookNode
from traitlets import Bool, Unicode
from typing import Tuple, Any
from textwrap import dedent


class OverwriteCells(NbGraderPreprocessor):
"""A preprocessor to overwrite information about grade and solution cells."""

add_missing_cells = Bool(
False,
help=dedent(
"""
Whether or not missing grade_cells should be added back
to the notebooks being graded.
"""
),
).tag(config=True)

missing_cell_notification = Unicode(
"This cell (id:{cell_id}) was missing from the submission. " +
"It was added back by nbgrader.\n\n", # Markdown requires two newlines
help=dedent(
"""
A text to add at the beginning of every missing cell re-added to the notebook during autograding.
"""
)
).tag(config=True)

def missing_cell_transform(self, source_cell, max_score, is_solution=False, is_task=False):
"""
Converts source_cell obtained from Gradebook into a cell that can be added to the notebook.
It is assumed that the cell is a grade_cell (unless is_task=True)
"""

missing_cell_notification = self.missing_cell_notification.format(cell_id=source_cell.name)

cell = {
"cell_type": source_cell.cell_type,
"metadata": {
"deletable": False,
"editable": False,
"nbgrader": {
"grade": True,
"grade_id": source_cell.name,
"locked": source_cell.locked,
"checksum": source_cell.checksum,
"cell_type": source_cell.cell_type,
"points": max_score,
"solution": False
}
},
"source": missing_cell_notification + source_cell.source
}

# Code cell format is slightly different
if cell["cell_type"] == "code":
cell["execution_count"] = None
cell["outputs"] = []
cell["source"] = "# " + cell["source"] # make the notification we add a comment

# some grade cells are editable (manually graded answers)
if is_solution:
del cell["metadata"]["editable"]
cell["metadata"]["nbgrader"]["solution"] = True
# task cells are also a bit different
elif is_task:
cell["metadata"]["nbgrader"]["grade"] = False
cell["metadata"]["nbgrader"]["task"] = True
# this is when task cells were added so metadata validation should start from here
cell["metadata"]["nbgrader"]["schema_version"] = 3

cell = NotebookNode(cell)
cell = MetadataValidator().upgrade_cell_metadata(cell)
return cell

def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
# pull information from the resources
self.notebook_id = resources['nbgrader']['notebook']
Expand All @@ -22,6 +92,77 @@ def preprocess(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[Notebo

with self.gradebook:
nb, resources = super(OverwriteCells, self).preprocess(nb, resources)
if self.add_missing_cells:
nb, resources = self.add_missing_grade_cells(nb, resources)
nb, resources = self.add_missing_task_cells(nb, resources)

return nb, resources

def add_missing_grade_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
"""
Add missing grade cells back to the notebook.
If missing, find the previous solution/grade cell, and add the current cell after it.
It is assumed such a cell exists because
presumably the grade_cell exists to grade some work in the solution cell.
"""
source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id)
source_cells = source_nb.source_cells
source_cell_ids = [cell.name for cell in source_cells]
grade_cells = {cell.name: cell for cell in source_nb.grade_cells}
solution_cell_ids = [cell.name for cell in source_nb.solution_cells]

# track indices of solution and grade cells in the submitted notebook
submitted_cell_idxs = dict()
for idx, cell in enumerate(nb.cells):
if utils.is_grade(cell) or utils.is_solution(cell):
submitted_cell_idxs[cell.metadata.nbgrader["grade_id"]] = idx

# Every time we add a cell, the idxs above get shifted
# We could process the notebook backwards, but that makes adding the cells in the right place more difficult
# So we keep track of how many we have added so far
added_count = 0

for grade_cell_id, grade_cell in grade_cells.items():
# If missing, find the previous solution/grade cell, and add the current cell after it.
if grade_cell_id not in submitted_cell_idxs:
self.log.warning(f"Missing grade cell {grade_cell_id} encountered, adding to notebook")
source_cell_idx = source_cell_ids.index(grade_cell_id)
cell_to_add = source_cells[source_cell_idx]
cell_to_add = self.missing_cell_transform(cell_to_add, grade_cell.max_score,
is_solution=grade_cell_id in solution_cell_ids)
# First cell was deleted, add it to start
if source_cell_idx == 0:
nb.cells.insert(0, cell_to_add)
submitted_cell_idxs[grade_cell_id] = 0
# Deleted cell is not the first, add it after the previous solution/grade cell
else:
prev_cell_id = source_cell_ids[source_cell_idx - 1]
prev_cell_idx = submitted_cell_idxs[prev_cell_id] + added_count
nb.cells.insert(prev_cell_idx + 1, cell_to_add) # +1 to add it after
submitted_cell_idxs[grade_cell_id] = submitted_cell_idxs[prev_cell_id]

# If the cell we just added is followed by other missing cells, we need to know its index in the nb
# However, no need to add `added_count` to avoid double-counting

added_count += 1 # shift idxs

return nb, resources

def add_missing_task_cells(self, nb: NotebookNode, resources: ResourcesDict) -> Tuple[NotebookNode, ResourcesDict]:
"""
Add missing task cells back to the notebook.
We can't figure out their original location, so they are added at the end, in their original order.
"""
source_nb = self.gradebook.find_notebook(self.notebook_id, self.assignment_id)
source_cells = source_nb.source_cells
source_cell_ids = [cell.name for cell in source_cells]
submitted_ids = [cell["metadata"]["nbgrader"]["grade_id"] for cell in nb.cells if
"nbgrader" in cell["metadata"]]
for task_cell in source_nb.task_cells:
if task_cell.name not in submitted_ids:
cell_to_add = source_cells[source_cell_ids.index(task_cell.name)]
cell_to_add = self.missing_cell_transform(cell_to_add, task_cell.max_score, is_task=True)
nb.cells.append(cell_to_add)

return nb, resources

Expand Down
56 changes: 53 additions & 3 deletions nbgrader/tests/preprocessors/test_overwritecells.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
import pytest

from nbformat.v4 import new_notebook
from nbformat.v4 import new_notebook, new_markdown_cell

from ...preprocessors import SaveCells, OverwriteCells
from ...api import Gradebook
from ...utils import compute_checksum
from .base import BaseTestPreprocessor
from .. import (
create_grade_cell, create_solution_cell, create_grade_and_solution_cell,
create_locked_cell)
create_locked_cell, create_task_cell)


@pytest.fixture
Expand All @@ -23,6 +23,7 @@ def gradebook(request, db):

def fin():
gb.close()

request.addfinalizer(fin)

return gb
Expand Down Expand Up @@ -197,7 +198,7 @@ def test_overwrite_locked_checksum(self, preprocessors, resources):

assert cell.metadata.nbgrader["checksum"] == compute_checksum(cell)

def test_nonexistant_grade_id(self, preprocessors, resources):
def test_nonexistent_grade_id(self, preprocessors, resources):
"""Are cells not in the database ignored?"""
cell = create_grade_cell("", "code", "", 1)
cell.metadata.nbgrader['grade'] = False
Expand All @@ -215,3 +216,52 @@ def test_nonexistant_grade_id(self, preprocessors, resources):
nb, resources = preprocessors[1].preprocess(nb, resources)
assert 'grade_id' not in cell.metadata.nbgrader

# Tests for adding missing cells back
def test_add_missing_cells(self, preprocessors, resources):
"""
Note: This test will produce warnings (from OverwriteCells preprocessor) by default.
Current implementation of adding missing cells should:
- add missing cells right after the previous grade/solution cell, as the best approximation of their location
- add task cells at the end (because we can't detect their location in the notebook), in order of appearance
"""

cells = [
create_solution_cell("Code assignment", "code", "code_solution"),
create_grade_cell("some tests", "code", "code_test1", 1),
create_grade_cell("more tests", "code", "code_test2", 1),
new_markdown_cell("some description"),
create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded1", 1),
create_grade_and_solution_cell("write answer here", "markdown", "md_manually_graded2", 1),
new_markdown_cell("some description"),
create_task_cell("some task description", "markdown", "task_cell1", 1),
new_markdown_cell("some description"),
create_task_cell("some task description", "markdown", "task_cell2", 1),
]
# Add checksums to suppress warning
nbgrader_cells = [0, 1, 2, 4, 5, 7, 9]
for idx, cell in enumerate(cells):
if idx in nbgrader_cells:
cell.metadata.nbgrader["checksum"] = compute_checksum(cell)

expected_order = [0, 1, 2, 4, 5, 3, 6, 8, 7, 9]
expected = [cells[i].metadata.nbgrader["grade_id"] if "nbgrader" in cells[i].metadata else "markdown" for i in expected_order]

nb = new_notebook()
nb.cells = cells

# save to database
nb, resources = preprocessors[0].preprocess(nb, resources)

# remove grade/task cells to test their restoration
nb.cells.pop(9)
nb.cells.pop(7)
nb.cells.pop(5)
nb.cells.pop(4)
nb.cells.pop(2)
nb.cells.pop(1)

# restore
preprocessors[1].add_missing_cells = True
nb, resources = preprocessors[1].preprocess(nb, resources)
result = [cell["metadata"]["nbgrader"]["grade_id"] if "nbgrader" in cell["metadata"] else "markdown" for cell in nb.cells]
assert expected == result
Loading