-
Notifications
You must be signed in to change notification settings - Fork 49
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Remove bonus xp #1089
Remove bonus xp #1089
Changes from all commits
bab5d39
c7e13ed
c65fa09
998b73e
27c3a6f
527619f
1f407cc
ee9845d
4bb69dc
e3e4be0
48ac193
126d18f
ef76f3d
dd8b7b8
da4e11f
6bf4626
e561ab7
848e538
88324a1
189f4dd
c9549c2
f3c2347
246d3ac
44670b5
fbe2e34
7e2b154
3930e4e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -916,7 +916,7 @@ | |
raw_answer, | ||
force_submit | ||
) do | ||
with {:ok, team} <- find_team(question.assessment.id, cr_id), | ||
{:ok, submission} <- find_or_create_submission(cr, question.assessment), | ||
{:status, true} <- {:status, force_submit or submission.status != :submitted}, | ||
{:ok, _answer} <- insert_or_update_answer(submission, question, raw_answer, cr_id) do | ||
|
@@ -1019,8 +1019,8 @@ | |
|
||
def finalise_submission(submission = %Submission{}) do | ||
with {:status, :attempted} <- {:status, submission.status}, | ||
{:ok, updated_submission} <- update_submission_status_and_xp_bonus(submission) do | ||
# Couple with update_submission_status_and_xp_bonus to ensure notification is sent | ||
{:ok, updated_submission} <- update_submission_status(submission) do | ||
# Couple with update_submission_status and update_xp_bonus to ensure notification is sent | ||
Notifications.write_notification_when_student_submits(submission) | ||
# Send email notification to avenger | ||
%{notification_type: "assessment_submission", submission_id: updated_submission.id} | ||
|
@@ -1029,6 +1029,7 @@ | |
|
||
# Begin autograding job | ||
GradingJob.force_grade_individual_submission(updated_submission) | ||
update_xp_bonus(submission) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this should not be called here. A better place to trigger this is after the grading job response. Also remember to remove irrelevant test in the cadet web and add relevant test in cadet and grading job. |
||
|
||
{:ok, nil} | ||
else | ||
|
@@ -1413,30 +1414,75 @@ | |
end | ||
end | ||
|
||
@spec update_submission_status_and_xp_bonus(Submission.t()) :: | ||
@spec update_submission_status(Submission.t()) :: | ||
{:ok, Submission.t()} | {:error, Ecto.Changeset.t()} | ||
defp update_submission_status_and_xp_bonus(submission = %Submission{}) do | ||
assessment = submission.assessment | ||
assessment_conifg = Repo.get_by(AssessmentConfig, id: assessment.config_id) | ||
defp update_submission_status(submission = %Submission{}) do | ||
submission | ||
|> Submission.changeset(%{status: :submitted, submitted_at: Timex.now()}) | ||
|> Repo.update() | ||
end | ||
|
||
max_bonus_xp = assessment_conifg.early_submission_xp | ||
early_hours = assessment_conifg.hours_before_early_xp_decay | ||
@spec update_xp_bonus(Submission.t()) :: | ||
{:ok, Submission.t()} | {:error, Ecto.Changeset.t()} | ||
defp update_xp_bonus(submission = %Submission{id: submission_id}) do | ||
# to ensure backwards compatibility | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what do you mean here? what happen if There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the reason this was here was because I was worried that if a grader went to grade an old submission where submitted_at is null, the xp_bonus will be set to 0 and decrease. Therefore if xp_bonus is not 0 it does nothing and keeps it at whatever value it was previously set to. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. but, if you have set the The way to reason this is that think about this as a function, the bonus_xp should be a pure mathematical function of ( Otherwise, if the autograder thinks that it is not low effort and bonus_xp is awarded, then the our staff grader will not be able to revoke this low effort 'bonus_xp'. If we decided that the TA should not affect the bonus xp behaviour, then we should not trigger update in update grading info. |
||
if submission.xp_bonus == 0 do | ||
assessment = submission.assessment | ||
assessment_conifg = Repo.get_by(AssessmentConfig, id: assessment.config_id) | ||
|
||
xp_bonus = | ||
if Timex.before?(Timex.now(), Timex.shift(assessment.open_at, hours: early_hours)) do | ||
max_bonus_xp | ||
else | ||
# This logic interpolates from max bonus at early hour to 0 bonus at close time | ||
decaying_hours = Timex.diff(assessment.close_at, assessment.open_at, :hours) - early_hours | ||
remaining_hours = Enum.max([0, Timex.diff(assessment.close_at, Timex.now(), :hours)]) | ||
proportion = if(decaying_hours > 0, do: remaining_hours / decaying_hours, else: 1) | ||
bonus_xp = round(max_bonus_xp * proportion) | ||
Enum.max([0, bonus_xp]) | ||
end | ||
max_bonus_xp = assessment_conifg.early_submission_xp | ||
early_hours = assessment_conifg.hours_before_early_xp_decay | ||
|
||
submission | ||
|> Submission.changeset(%{status: :submitted, xp_bonus: xp_bonus}) | ||
|> Repo.update() | ||
ans_xp = | ||
Answer | ||
|> where(submission_id: ^submission_id) | ||
|> order_by(:question_id) | ||
|> group_by([a], a.id) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You will only have answers here, there is no need to group |
||
|> select([a], %{ | ||
# grouping by submission, so s.xp_bonus will be the same, but we need an | ||
# aggregate function | ||
total_xp: sum(a.xp) + sum(a.xp_adjustment) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. rethink about what you are querying before editing the code. As mentioned, there is only one answer and there is not need to sum. |
||
}) | ||
|
||
total = | ||
ans_xp | ||
|> subquery | ||
|> select([a], %{ | ||
total_xp: sum(a.total_xp) | ||
}) | ||
|> Repo.one() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this sum does nothing, since you did not group by anything. |
||
|
||
xp = decimal_to_integer(total.total_xp) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. if you just want to get the current xp, there should be shorter and cleaner way to do this. |
||
|
||
cur_time = | ||
if submission.submitted_at == nil do | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. as mentioned, it should not be nil, and this should be in the guard. |
||
Timex.now() | ||
else | ||
submission.submitted_at | ||
end | ||
|
||
xp_bonus = | ||
if xp <= 0 do | ||
0 | ||
else | ||
if Timex.before?(cur_time, Timex.shift(assessment.open_at, hours: early_hours)) do | ||
max_bonus_xp | ||
else | ||
# This logic interpolates from max bonus at early hour to 0 bonus at close time | ||
decaying_hours = | ||
Timex.diff(assessment.close_at, assessment.open_at, :hours) - early_hours | ||
|
||
remaining_hours = Enum.max([0, Timex.diff(assessment.close_at, cur_time, :hours)]) | ||
proportion = if(decaying_hours > 0, do: remaining_hours / decaying_hours, else: 1) | ||
bonus_xp = round(max_bonus_xp * proportion) | ||
Enum.max([0, bonus_xp]) | ||
end | ||
end | ||
|
||
submission | ||
|> Submission.changeset(%{xp_bonus: xp_bonus}) | ||
|> Repo.update() | ||
end | ||
end | ||
|
||
defp update_submission_status_router(submission = %Submission{}, question = %Question{}) do | ||
|
@@ -2231,12 +2277,19 @@ | |
|
||
is_own_submission = grader_id == answer.submission.student_id | ||
|
||
submission = | ||
Submission | ||
|> join(:inner, [s], a in assoc(s, :assessment)) | ||
|> preload([_, a], assessment: a) | ||
|> Repo.get(submission_id) | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. getting the submission and updating the bonus, both are not guarded here. Should write error handling here. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. like how it was done with the |
||
with {:answer_found?, true} <- {:answer_found?, is_map(answer)}, | ||
{:status, true} <- | ||
{:status, answer.submission.status == :submitted or is_own_submission}, | ||
{:valid, changeset = %Ecto.Changeset{valid?: true}} <- | ||
{:valid, Answer.grading_changeset(answer, attrs)}, | ||
{:ok, _} <- Repo.update(changeset) do | ||
update_xp_bonus(submission) | ||
{:ok, nil} | ||
else | ||
{:answer_found?, false} -> | ||
|
@@ -2501,7 +2554,7 @@ | |
|
||
def has_last_modified_answer?( | ||
question = %Question{}, | ||
cr = %CourseRegistration{id: cr_id}, | ||
last_modified_at, | ||
force_submit | ||
) do | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
defmodule Cadet.Repo.Migrations.SubmissionsAddSubmittedAtColumn do | ||
use Ecto.Migration | ||
|
||
def change do | ||
alter table(:submissions) do | ||
add(:submitted_at, :timestamp, null: true) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. try to enforce a constraint like: Please also update the auto-submission job to insert this value appropriately. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. remember to write data migration, to make existing data conform to this constraint. |
||
end | ||
end | ||
end |
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please additionally do unit test. since now update status and update bonus are separated, they should be tested individually. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. please note that in this writing, the mock is only use to test if certain function is called in the process. it does not test what your algorithm will react to different grading job response. |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1069,7 +1069,7 @@ defmodule CadetWeb.AssessmentsControllerTest do | |
end | ||
end | ||
|
||
test "submission of answer within early hours(seeded 48) of opening grants full XP bonus", %{ | ||
test "submission of answer with no effort grants 0 XP bonus", %{ | ||
conn: conn, | ||
courses: %{course1: course1}, | ||
role_crs: role_crs | ||
|
@@ -1117,6 +1117,194 @@ defmodule CadetWeb.AssessmentsControllerTest do | |
|
||
submission_db = Repo.get(Submission, submission.id) | ||
|
||
assert submission_db.status == :submitted | ||
assert submission_db.xp_bonus == 0 | ||
end | ||
end | ||
|
||
test "submission of answer grants XP bonus only after being marked by an avenger", %{ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. is this the behaviour that we want? what if it is autograded? |
||
conn: conn, | ||
courses: %{course1: course1}, | ||
role_crs: role_crs | ||
} do | ||
with_mock GradingJob, force_grade_individual_submission: fn _ -> nil end do | ||
assessment_config = | ||
insert( | ||
:assessment_config, | ||
early_submission_xp: 100, | ||
hours_before_early_xp_decay: 48, | ||
course: course1 | ||
) | ||
|
||
assessment = | ||
insert( | ||
:assessment, | ||
open_at: Timex.shift(Timex.now(), hours: -40), | ||
close_at: Timex.shift(Timex.now(), days: 7), | ||
is_published: true, | ||
config: assessment_config, | ||
course: course1 | ||
) | ||
|
||
question = insert(:programming_question, assessment: assessment) | ||
|
||
group = insert(:group, leader: role_crs.staff) | ||
|
||
course_reg = | ||
insert(:course_registration, %{role: :student, group: group, course: course1}) | ||
|
||
submission = | ||
insert(:submission, assessment: assessment, student: course_reg, status: :attempted) | ||
|
||
insert( | ||
:answer, | ||
submission: submission, | ||
question: question, | ||
answer: %{code: "f => f(f);"}, | ||
xp_adjustment: 0 | ||
) | ||
|
||
conn | ||
|> sign_in(course_reg.user) | ||
|> post(build_url_submit(course1.id, assessment.id)) | ||
|> response(200) | ||
|
||
submission_db = Repo.get(Submission, submission.id) | ||
|
||
assert submission_db.status == :submitted | ||
assert submission_db.xp_bonus == 0 | ||
|
||
grade_question = fn question -> | ||
Assessments.update_grading_info( | ||
%{submission_id: submission.id, question_id: question.id}, | ||
%{"xp_adjustment" => 10}, | ||
role_crs.staff | ||
) | ||
end | ||
|
||
grade_question.(question) | ||
|
||
submission_db = Repo.get(Submission, submission.id) | ||
assert submission_db.xp_bonus == 100 | ||
end | ||
end | ||
|
||
test "submission of answer grants 0 XP bonus if an avenger gives 0 as well", %{ | ||
conn: conn, | ||
courses: %{course1: course1}, | ||
role_crs: role_crs | ||
} do | ||
with_mock GradingJob, force_grade_individual_submission: fn _ -> nil end do | ||
assessment_config = | ||
insert( | ||
:assessment_config, | ||
early_submission_xp: 100, | ||
hours_before_early_xp_decay: 48, | ||
course: course1 | ||
) | ||
|
||
assessment = | ||
insert( | ||
:assessment, | ||
open_at: Timex.shift(Timex.now(), hours: -40), | ||
close_at: Timex.shift(Timex.now(), days: 7), | ||
is_published: true, | ||
config: assessment_config, | ||
course: course1 | ||
) | ||
|
||
question = insert(:programming_question, assessment: assessment) | ||
|
||
group = insert(:group, leader: role_crs.staff) | ||
|
||
course_reg = | ||
insert(:course_registration, %{role: :student, group: group, course: course1}) | ||
|
||
submission = | ||
insert(:submission, assessment: assessment, student: course_reg, status: :attempted) | ||
|
||
insert( | ||
:answer, | ||
submission: submission, | ||
question: question, | ||
answer: %{code: "f => f(f);"}, | ||
xp_adjustment: 0 | ||
) | ||
|
||
conn | ||
|> sign_in(course_reg.user) | ||
|> post(build_url_submit(course1.id, assessment.id)) | ||
|> response(200) | ||
|
||
submission_db = Repo.get(Submission, submission.id) | ||
|
||
assert submission_db.status == :submitted | ||
assert submission_db.xp_bonus == 0 | ||
|
||
grade_question = fn question -> | ||
Assessments.update_grading_info( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This test should not be here, if you want to run this function, you should test it in Additionally, the admin grading controller endpoint which is affected by the |
||
%{submission_id: submission.id, question_id: question.id}, | ||
%{"xp_adjustment" => 0}, | ||
role_crs.staff | ||
) | ||
end | ||
|
||
grade_question.(question) | ||
|
||
submission_db = Repo.get(Submission, submission.id) | ||
assert submission_db.xp_bonus == 0 | ||
end | ||
end | ||
|
||
test "submission of answer within early hours(seeded 48) of opening grants full XP bonus", %{ | ||
conn: conn, | ||
courses: %{course1: course1}, | ||
role_crs: role_crs | ||
} do | ||
with_mock GradingJob, force_grade_individual_submission: fn _ -> nil end do | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. some of these seems to be old issue since |
||
assessment_config = | ||
insert( | ||
:assessment_config, | ||
early_submission_xp: 100, | ||
hours_before_early_xp_decay: 48, | ||
course: course1 | ||
) | ||
|
||
assessment = | ||
insert( | ||
:assessment, | ||
open_at: Timex.shift(Timex.now(), hours: -40), | ||
close_at: Timex.shift(Timex.now(), days: 7), | ||
is_published: true, | ||
config: assessment_config, | ||
course: course1 | ||
) | ||
|
||
question = insert(:programming_question, assessment: assessment) | ||
|
||
group = insert(:group, leader: role_crs.staff) | ||
|
||
course_reg = | ||
insert(:course_registration, %{role: :student, group: group, course: course1}) | ||
|
||
submission = | ||
insert(:submission, assessment: assessment, student: course_reg, status: :attempted) | ||
|
||
insert( | ||
:answer, | ||
submission: submission, | ||
question: question, | ||
answer: %{code: "f => f(f);"}, | ||
xp: 10 | ||
) | ||
|
||
conn | ||
|> sign_in(course_reg.user) | ||
|> post(build_url_submit(course1.id, assessment.id)) | ||
|> response(200) | ||
|
||
submission_db = Repo.get(Submission, submission.id) | ||
|
||
assert submission_db.status == :submitted | ||
assert submission_db.xp_bonus == 100 | ||
end | ||
|
@@ -1161,7 +1349,8 @@ defmodule CadetWeb.AssessmentsControllerTest do | |
:answer, | ||
submission: submission, | ||
question: question, | ||
answer: %{code: "f => f(f);"} | ||
answer: %{code: "f => f(f);"}, | ||
xp: 10 | ||
) | ||
|
||
conn | ||
|
@@ -1219,7 +1408,8 @@ defmodule CadetWeb.AssessmentsControllerTest do | |
:answer, | ||
submission: submission, | ||
question: question, | ||
answer: %{code: "f => f(f);"} | ||
answer: %{code: "f => f(f);"}, | ||
xp: 10 | ||
) | ||
|
||
conn | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is
update_xp_bonus
still coupled?