Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test fix: do not depend on map order #399

Merged
merged 4 commits into from
Oct 13, 2023
Merged

Test fix: do not depend on map order #399

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0cb160c
update submodule
jiegillet Oct 12, 2023
e186f52
compare decoded json
jiegillet Oct 12, 2023
ba75858
woops
jiegillet Oct 12, 2023
c76a0c6
order compiler warnings
jiegillet Oct 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion elixir
Open in desktop
Submodule elixir updated 2 files
+1 −1 exercises/concept/language-list/.meta/config.json
+4 −2 exercises/practice/phone-number/.docs/instructions.md
2 changes: 1 addition & 1 deletion lib/elixir_analyzer/exercise_test/common_checks/compiler_warnings.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ defmodule ElixirAnalyzer.ExerciseTest.CommonChecks.CompilerWarnings do
type: :actionable,
name: Constants.solution_compiler_warnings(),
comment: Constants.solution_compiler_warnings(),
params: %{warnings: Enum.map_join(warnings, &format_warning/1)}
params: %{warnings: warnings |> Enum.sort() |> Enum.map_join(&format_warning/1)}
}}
]
end
Expand Down
251 changes: 207 additions & 44 deletions test/elixir_analyzer_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,13 @@ defmodule ElixirAnalyzerTest do
path = "./test_data/two_fer/perfect_solution/"
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}
"""
expected_output =
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "referred solution with comments" do
Expand All @@ -28,9 +30,53 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.two-fer.use_of_function_header\"},{\"type\":\"actionable\",\"comment\":\"elixir.solution.use_specification\"},{\"type\":\"actionable\",\"comment\":\"elixir.solution.raise_fn_clause_error\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"_nameInPascalCase\",\"expected\":\"_name_in_pascal_case\"},\"comment\":\"elixir.solution.variable_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"someUnusedModuleAttribute\",\"expected\":\"some_unused_module_attribute\"},\"comment\":\"elixir.solution.module_attribute_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"My_empty_module\",\"expected\":\"MyEmptyModule\"},\"comment\":\"elixir.solution.module_pascal_case\"},{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: module attribute @someUnusedModuleAttribute was set but never used\\n lib/two_fer.ex:2\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.use_module_doc\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.indentation\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{"comment" => "elixir.two-fer.use_of_function_header", "type" => "actionable"},
%{"comment" => "elixir.solution.use_specification", "type" => "actionable"},
%{"comment" => "elixir.solution.raise_fn_clause_error", "type" => "actionable"},
%{
"comment" => "elixir.solution.variable_name_snake_case",
"params" => %{"actual" => "_nameInPascalCase", "expected" => "_name_in_pascal_case"},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.module_attribute_name_snake_case",
"params" => %{
"actual" => "someUnusedModuleAttribute",
"expected" => "some_unused_module_attribute"
},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.module_pascal_case",
"params" => %{"actual" => "My_empty_module", "expected" => "MyEmptyModule"},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: module attribute @someUnusedModuleAttribute was set but never used\n lib/two_fer.ex:2\n\n"
},
"type" => "actionable"
},
%{"comment" => "elixir.solution.use_module_doc", "type" => "informative"},
%{"comment" => "elixir.solution.indentation", "type" => "informative"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
"type" => "informative"
},
%{"comment" => "elixir.general.feedback_request", "type" => "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "solution with informative comments only" do
Expand All @@ -40,9 +86,14 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"informative\",\"comment\":\"elixir.solution.use_module_doc\"}],\"summary\":\"Check the comments for some things to learn. 📖\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{"comment" => "elixir.solution.use_module_doc", "type" => "informative"}
],
"summary" => "Check the comments for some things to learn. 📖"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "error solution" do
Expand All @@ -52,11 +103,22 @@ defmodule ElixirAnalyzerTest do
assert capture_log(fn ->
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[{\"type\":\"essential\",\"params\":{\"error\":\"missing terminator: end (for \\\"do\\\" starting at line 1)\",\"line\":14},\"comment\":\"elixir.general.parsing_error\"}],\"summary\":\"Check the comments for things to fix. 🛠\"}
"""

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
expected_output =
%{
"comments" => [
%{
"comment" => "elixir.general.parsing_error",
"params" => %{
"error" => "missing terminator: end (for \"do\" starting at line 1)",
"line" => 14
},
"type" => "essential"
}
],
"summary" => "Check the comments for things to fix. 🛠"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file could not be parsed."
end

Expand All @@ -67,11 +129,22 @@ defmodule ElixirAnalyzerTest do
assert capture_log(fn ->
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output = """
{\"comments\":[{\"type\":\"essential\",\"params\":{\"file_name\":\"two_fer.ex\",\"path\":\"test_data/two_fer/missing_file_solution/\"},\"comment\":\"elixir.general.file_not_found\"}],\"summary\":\"Check the comments for things to fix. 🛠\"}
"""

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
expected_output =
%{
"comments" => [
%{
"comment" => "elixir.general.file_not_found",
"params" => %{
"file_name" => "two_fer.ex",
"path" => "test_data/two_fer/missing_file_solution/"
},
"type" => "essential"
}
],
"summary" => "Check the comments for things to fix. 🛠"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Code file not found. Reason: enoent"
end

Expand All @@ -91,9 +164,50 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.solution.raise_fn_clause_error\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"_nameInPascalCase\",\"expected\":\"_name_in_pascal_case\"},\"comment\":\"elixir.solution.variable_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"someUnusedModuleAttribute\",\"expected\":\"some_unused_module_attribute\"},\"comment\":\"elixir.solution.module_attribute_name_snake_case\"},{\"type\":\"actionable\",\"params\":{\"actual\":\"My_empty_module\",\"expected\":\"MyEmptyModule\"},\"comment\":\"elixir.solution.module_pascal_case\"},{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: module attribute @someUnusedModuleAttribute was set but never used\\n lib/two_fer.ex:2\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.indentation\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
%{
"comments" => [
%{"comment" => "elixir.solution.raise_fn_clause_error", "type" => "actionable"},
%{
"comment" => "elixir.solution.variable_name_snake_case",
"params" => %{"actual" => "_nameInPascalCase", "expected" => "_name_in_pascal_case"},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.module_attribute_name_snake_case",
"params" => %{
"actual" => "someUnusedModuleAttribute",
"expected" => "some_unused_module_attribute"
},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.module_pascal_case",
"params" => %{"actual" => "My_empty_module", "expected" => "MyEmptyModule"},
"type" => "actionable"
},
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: module attribute @someUnusedModuleAttribute was set but never used\n lib/two_fer.ex:2\n\n"
},
"type" => "actionable"
},
%{"comment" => "elixir.solution.indentation", "type" => "informative"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
"type" => "informative"
},
%{"comment" => "elixir.general.feedback_request", "type" => "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end
end

Expand All @@ -114,9 +228,14 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"celebratory\",\"comment\":\"elixir.solution.same_as_exemplar\"}],\"summary\":\"You're doing something right. 🎉\"}"

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
%{
"comments" => [
%{"comment" => "elixir.solution.same_as_exemplar", "type" => "celebratory"}
],
"summary" => "You're doing something right. 🎉"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "perfect solution for exercise with multiple solution files" do
Expand All @@ -125,9 +244,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "failing solution with comments" do
Expand All @@ -136,9 +258,24 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"comment\":\"elixir.lasagna.function_reuse\"},{\"type\":\"informative\",\"params\":{\"actual\":\"def public_helper(_)\",\"expected\":\"defp public_helper(_)\"},\"comment\":\"elixir.solution.private_helper_functions\"},{\"type\":\"informative\",\"comment\":\"elixir.solution.todo_comment\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{"comment" => "elixir.lasagna.function_reuse", "type" => "actionable"},
%{
"comment" => "elixir.solution.private_helper_functions",
"params" => %{
"actual" => "def public_helper(_)",
"expected" => "defp public_helper(_)"
},
"type" => "informative"
},
%{"comment" => "elixir.solution.todo_comment", "type" => "informative"},
%{"comment" => "elixir.general.feedback_request", "type" => "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "failing solution that uses deprecated modules" do
Expand All @@ -147,9 +284,22 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[{\"type\":\"actionable\",\"params\":{\"warnings\":\"warning: HashDict.new/0 is deprecated. Use maps and the Map module instead\\n lib/lasagna.ex:7\\n\\nwarning: HashSet.member?/2 is deprecated. Use the MapSet module instead\\n lib/lasagna.ex:12\\n\\nwarning: HashSet.new/0 is deprecated. Use the MapSet module instead\\n lib/lasagna.ex:12\\n\\nwarning: Behaviour.defcallback/1 is deprecated. Use the @callback module attribute instead\\n lib/lasagna.ex:4\\n\\n\"},\"comment\":\"elixir.solution.compiler_warnings\"},{\"type\":\"informative\",\"comment\":\"elixir.general.feedback_request\"}],\"summary\":\"Check the comments for some suggestions. 📣\"}"

assert Submission.to_json(analyzed_exercise) == expected_output
%{
"comments" => [
%{
"comment" => "elixir.solution.compiler_warnings",
"params" => %{
"warnings" =>
"warning: Behaviour.defcallback/1 is deprecated. Use the @callback module attribute instead\n lib/lasagna.ex:4\n\nwarning: HashDict.new/0 is deprecated. Use maps and the Map module instead\n lib/lasagna.ex:7\n\nwarning: HashSet.member?/2 is deprecated. Use the MapSet module instead\n lib/lasagna.ex:12\n\nwarning: HashSet.new/0 is deprecated. Use the MapSet module instead\n lib/lasagna.ex:12\n\n"
},
"type" => "actionable"
},
%{"comment" => "elixir.general.feedback_request", "type" => "informative"}
],
"summary" => "Check the comments for some suggestions. 📣"
}

assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end

test "solution with missing exemplar" do
Expand All @@ -160,9 +310,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file not found. Reason: enoent"
end

Expand All @@ -174,9 +327,12 @@ defmodule ElixirAnalyzerTest do
analyzed_exercise = ElixirAnalyzer.analyze_exercise(exercise, path, path, @options)

expected_output =
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(analyzed_exercise) == String.trim(expected_output)
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() == expected_output
end) =~ "Exemploid file could not be parsed."
end
end
Expand All @@ -196,11 +352,14 @@ defmodule ElixirAnalyzerTest do
Output written to ... a/b
"""

assert Submission.to_json(submission) ==
"{\"comments\":[],\"summary\":\"Submission analyzed. No automated suggestions found.\"}"
assert Submission.to_json(submission) |> Jason.decode!() ==
%{
"comments" => [],
"summary" => "Submission analyzed. No automated suggestions found."
}

assert Submission.to_json(%{submission | halted: true}) ==
"{\"comments\":[],\"summary\":\"Analysis was halted.\"}"
assert Submission.to_json(%{submission | halted: true}) |> Jason.decode!() ==
%{"comments" => [], "summary" => "Analysis was halted."}
end

test "solution with wrong analysis module" do
Expand Down Expand Up @@ -231,8 +390,12 @@ defmodule ElixirAnalyzerTest do
Output written to ... a/b
"""

assert Submission.to_json(analyzed_exercise) ==
"{\"comments\":[],\"summary\":\"Analysis was halted. Analysis skipped, unexpected error Elixir.ArgumentError\"}"
assert Submission.to_json(analyzed_exercise) |> Jason.decode!() ==
%{
"comments" => [],
"summary" =>
"Analysis was halted. Analysis skipped, unexpected error Elixir.ArgumentError"
}
end) =~ "[error] Loading exercise test suite 'Elixir.NonSense' failed"
end

Expand Down
Loading