Skip to content

Commit 2feac7f

Browse files
m-messerm-messer
committed
Refactored /preview into /evaluate workflow, removed /preview endpoint, added preSubmissionFeedback support to evaluations, and updated relevant tests and handlers.
1 parent 2eb6574 commit 2feac7f

3 files changed

Lines changed: 65 additions & 52 deletions

File tree

‎handler.py‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -74,12 +74,6 @@ def handle_muEd_command(event: JsonType, command: str) -> HandlerResponse:
7474
response = commands.evaluate_muEd(body)
7575
validate.body(response, MuEdResBodyValidators.EVALUATION)
7676

77-
elif command == "preview":
78-
body = parse.body(event)
79-
validate.body(body, MuEdReqBodyValidators.EVALUATION)
80-
response = commands.preview_muEd(body)
81-
validate.body(response, MuEdResBodyValidators.EVALUATION)
82-
8377
elif command == "healthcheck":
8478
response = commands.healthcheck()
8579

@@ -113,8 +107,6 @@ def handler(event: JsonType, _=None) -> HandlerResponse:
113107
path = "/evaluate/health"
114108
elif raw_path.endswith("/evaluate"):
115109
path = "/evaluate"
116-
elif raw_path.endswith("/preview"):
117-
path = "/preview"
118110
else:
119111
path = raw_path
120112

@@ -125,9 +117,6 @@ def handler(event: JsonType, _=None) -> HandlerResponse:
125117
elif path == "/evaluate/health":
126118
return handle_muEd_command(event, "healthcheck")
127119

128-
elif path == "/preview":
129-
return handle_muEd_command(event, "preview")
130-
131120
else:
132121
headers = event.get("headers", dict())
133122
command = headers.get("command", "eval")

‎tests/mued_handling_test.py‎

Lines changed: 60 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -194,17 +194,24 @@ def setUp(self) -> None:
194194
commands.preview_function = lambda response, params: {
195195
"preview": {"latex": f"\\text{{{response}}}", "sympy": response}
196196
}
197+
commands.evaluation_function = lambda response, answer, params: {
198+
"is_correct": True, "feedback": "Well done."
199+
}
197200
return super().setUp()
198201

199202
def tearDown(self) -> None:
200203
os.environ.pop("SCHEMA_DIR", None)
201204
commands.preview_function = None
205+
commands.evaluation_function = None
202206
return super().tearDown()
203207

204208
def test_preview_returns_feedback_list(self):
205209
event = {
206-
"path": "/preview",
207-
"body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}},
210+
"path": "/evaluate",
211+
"body": {
212+
"submission": {"type": "MATH", "content": {"expression": "x+1"}},
213+
"preSubmissionFeedback": {"enabled": True},
214+
},
208215
}
209216

210217
response = handler(event)
@@ -214,8 +221,11 @@ def test_preview_returns_feedback_list(self):
214221

215222
def test_preview_feedback_id_is_preSubmissionFeedback(self):
216223
event = {
217-
"path": "/preview",
218-
"body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}},
224+
"path": "/evaluate",
225+
"body": {
226+
"submission": {"type": "MATH", "content": {"expression": "x+1"}},
227+
"preSubmissionFeedback": {"enabled": True},
228+
},
219229
}
220230

221231
response = handler(event)
@@ -224,8 +234,11 @@ def test_preview_feedback_id_is_preSubmissionFeedback(self):
224234

225235
def test_preview_contains_preSubmissionFeedback_field(self):
226236
event = {
227-
"path": "/preview",
228-
"body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}},
237+
"path": "/evaluate",
238+
"body": {
239+
"submission": {"type": "MATH", "content": {"expression": "x+1"}},
240+
"preSubmissionFeedback": {"enabled": True},
241+
},
229242
}
230243

231244
response = handler(event)
@@ -234,8 +247,11 @@ def test_preview_contains_preSubmissionFeedback_field(self):
234247

235248
def test_preview_preSubmissionFeedback_has_latex_and_sympy(self):
236249
event = {
237-
"path": "/preview",
238-
"body": {"submission": {"type": "MATH", "content": {"expression": "x+1"}}},
250+
"path": "/evaluate",
251+
"body": {
252+
"submission": {"type": "MATH", "content": {"expression": "x+1"}},
253+
"preSubmissionFeedback": {"enabled": True},
254+
},
239255
}
240256

241257
response = handler(event)
@@ -246,16 +262,19 @@ def test_preview_preSubmissionFeedback_has_latex_and_sympy(self):
246262

247263
def test_preview_missing_submission_returns_error(self):
248264
event = {
249-
"path": "/preview",
250-
"body": {"configuration": {"params": {}}},
265+
"path": "/evaluate",
266+
"body": {
267+
"configuration": {"params": {}},
268+
"preSubmissionFeedback": {"enabled": True},
269+
},
251270
}
252271

253272
response = handler(event)
254273

255274
self.assertIn("error", response)
256275

257276
def test_preview_bodyless_event_returns_error(self):
258-
event = {"path": "/preview", "random": "metadata"}
277+
event = {"path": "/evaluate", "random": "metadata"}
259278

260279
response = handler(event)
261280

@@ -267,14 +286,32 @@ def test_preview_bodyless_event_returns_error(self):
267286

268287
def test_preview_invalid_submission_type_returns_error(self):
269288
event = {
270-
"path": "/preview",
271-
"body": {"submission": {"type": "INVALID", "content": {}}},
289+
"path": "/evaluate",
290+
"body": {
291+
"submission": {"type": "INVALID", "content": {}},
292+
"preSubmissionFeedback": {"enabled": True},
293+
},
272294
}
273295

274296
response = handler(event)
275297

276298
self.assertIn("error", response)
277299

300+
def test_presubmission_disabled_runs_normal_evaluation(self):
301+
event = {
302+
"path": "/evaluate",
303+
"body": {
304+
"submission": {"type": "MATH", "content": {"expression": "x+1"}},
305+
"preSubmissionFeedback": {"enabled": False},
306+
},
307+
}
308+
309+
response = handler(event)
310+
311+
self.assertIsInstance(response, list)
312+
self.assertIn("awardedPoints", response[0]) # type: ignore
313+
self.assertNotIn("preSubmissionFeedback", response[0]) # type: ignore
314+
278315

279316
class TestMuEdPreviewExtraction(unittest.TestCase):
280317
def setUp(self) -> None:
@@ -297,9 +334,10 @@ def tearDown(self) -> None:
297334

298335
def test_math_submission_extracts_expression(self):
299336
event = {
300-
"path": "/preview",
337+
"path": "/evaluate",
301338
"body": {
302339
"submission": {"type": "MATH", "content": {"expression": "x+1"}},
340+
"preSubmissionFeedback": {"enabled": True},
303341
},
304342
}
305343

@@ -309,9 +347,10 @@ def test_math_submission_extracts_expression(self):
309347

310348
def test_text_submission_extracts_text(self):
311349
event = {
312-
"path": "/preview",
350+
"path": "/evaluate",
313351
"body": {
314352
"submission": {"type": "TEXT", "content": {"text": "hello"}},
353+
"preSubmissionFeedback": {"enabled": True},
315354
},
316355
}
317356

@@ -321,10 +360,11 @@ def test_text_submission_extracts_text(self):
321360

322361
def test_configuration_params_forwarded(self):
323362
event = {
324-
"path": "/preview",
363+
"path": "/evaluate",
325364
"body": {
326365
"submission": {"type": "MATH", "content": {"expression": "x+1"}},
327366
"configuration": {"params": {"strict_syntax": False, "is_latex": True}},
367+
"preSubmissionFeedback": {"enabled": True},
328368
},
329369
}
330370

@@ -334,9 +374,10 @@ def test_configuration_params_forwarded(self):
334374

335375
def test_no_task_required(self):
336376
event = {
337-
"path": "/preview",
377+
"path": "/evaluate",
338378
"body": {
339379
"submission": {"type": "MATH", "content": {"expression": "sin(x)"}},
380+
"preSubmissionFeedback": {"enabled": True},
340381
},
341382
}
342383

@@ -347,9 +388,10 @@ def test_no_task_required(self):
347388

348389
def test_preview_result_propagated(self):
349390
event = {
350-
"path": "/preview",
391+
"path": "/evaluate",
351392
"body": {
352393
"submission": {"type": "MATH", "content": {"expression": "x+1"}},
394+
"preSubmissionFeedback": {"enabled": True},
353395
},
354396
}
355397

‎tools/commands.py‎

Lines changed: 5 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,11 @@ def evaluate_muEd(body: JsonType) -> List[Dict]:
155155
response = submission.get("content", {}).get(content_key)
156156
params = body.get("configuration", {}).get("params", {})
157157

158+
pre_sub = body.get("preSubmissionFeedback") or {}
159+
if pre_sub.get("enabled"):
160+
preview_result = preview_function(response, params)
161+
return [{"preSubmissionFeedback": preview_result.get("preview", {})}]
162+
158163
task = body.get("task")
159164
if task:
160165
ref = task.get("referenceSolution") or {}
@@ -178,29 +183,6 @@ def evaluate_muEd(body: JsonType) -> List[Dict]:
178183
return [feedback_item]
179184

180185

181-
def preview_muEd(body: JsonType) -> List[Dict]:
182-
"""Run the preview command for the evaluation function (muEd format).
183-
184-
Args:
185-
body (JsonType): The validated muEd request body.
186-
187-
Returns:
188-
List[Dict]: A list containing a single Feedback item with preview data.
189-
"""
190-
submission = body["submission"]
191-
sub_type = submission.get("type", "MATH")
192-
_type_key = {"MATH": "expression", "TEXT": "text", "CODE": "code", "MODEL": "model"}
193-
content_key = _type_key.get(sub_type, "value")
194-
195-
response = submission.get("content", {}).get(content_key)
196-
params = body.get("configuration", {}).get("params", {})
197-
198-
preview_result = preview_function(response, params)
199-
return [{
200-
"preSubmissionFeedback": preview_result.get("preview", {}),
201-
}]
202-
203-
204186
def get_case_feedback(
205187
response: Any,
206188
params: Dict,

0 commit comments

Comments
 (0)