From 2426b8df13d9992d69af352db07f0934341a832a Mon Sep 17 00:00:00 2001 From: Glutexo Date: Fri, 26 Aug 2022 12:22:08 +0200 Subject: [PATCH 01/14] Introduce a Parser module stub I created a dummy module to start working on the Parser component. It lists all files in the current working directory with no extension. In a directory that only contains files owned by Onigumo, such files are those fetched by the Downloader. --- lib/onigumo.ex | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/lib/onigumo.ex b/lib/onigumo.ex index 5b2b697..bf35b83 100644 --- a/lib/onigumo.ex +++ b/lib/onigumo.ex @@ -57,3 +57,18 @@ defmodule Onigumo.Downloader do Application.get_env(:onigumo, :http_client) end end + +defmodule Onigumo.Parser do + @moduledoc """ + Web scraper + """ + + def main(root_path) do + root_path + |> File.ls!() + |> Enum.reject(&File.dir?(&1)) + |> Enum.reject(&String.contains?(&1, ".")) + |> Enum.join("\n") + |> IO.puts() + end +end From 10841b11cb1c9258bf53b9063926bc5a4a98ff95 Mon Sep 17 00:00:00 2001 From: dstroch Date: Fri, 7 Apr 2023 20:01:00 +0200 Subject: [PATCH 02/14] Move parser to a new file because of new rules set for naming structure --- lib/onigumo/downloader.ex | 15 --------------- lib/onigumo/parser.ex | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 15 deletions(-) create mode 100644 lib/onigumo/parser.ex diff --git a/lib/onigumo/downloader.ex b/lib/onigumo/downloader.ex index 7b88e6b..a581049 100644 --- a/lib/onigumo/downloader.ex +++ b/lib/onigumo/downloader.ex @@ -57,18 +57,3 @@ defmodule Onigumo.Downloader do Application.get_env(:onigumo, :http_client) end end - -defmodule Onigumo.Parser do - @moduledoc """ - Web scraper - """ - - def main(root_path) do - root_path - |> File.ls!() - |> Enum.reject(&File.dir?(&1)) - |> Enum.reject(&String.contains?(&1, ".")) - |> Enum.join("\n") - |> IO.puts() - end -end diff --git a/lib/onigumo/parser.ex b/lib/onigumo/parser.ex new file mode 100644 index 0000000..82ebd6d --- /dev/null +++ b/lib/onigumo/parser.ex @@ -0,0 +1,14 @@ +defmodule Onigumo.Parser do + @moduledoc """ + Web scraper + """ + + def main(root_path) do + root_path + |> File.ls!() + |> Enum.reject(&File.dir?(&1)) + |> Enum.reject(&String.contains?(&1, ".")) + |> Enum.join("\n") + |> IO.puts() + end +end From 65ddbfb5a01011db3f941ebfd2779428f8217bb9 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sat, 20 May 2023 19:11:54 +0200 Subject: [PATCH 03/14] List files with .raw extension Downloaded files now have .raw extesion, they are no longer without an extension at all. --- lib/onigumo/parser.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/onigumo/parser.ex b/lib/onigumo/parser.ex index 82ebd6d..7008404 100644 --- a/lib/onigumo/parser.ex +++ b/lib/onigumo/parser.ex @@ -6,8 +6,7 @@ defmodule Onigumo.Parser do def main(root_path) do root_path |> File.ls!() - |> Enum.reject(&File.dir?(&1)) - |> Enum.reject(&String.contains?(&1, ".")) + |> Enum.filter(fn filename -> Path.extname(filename) == ".raw" end) |> Enum.join("\n") |> IO.puts() end From 4b700f647282ae537ba926769b6f2fc3c2d67d50 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sat, 20 May 2023 19:13:00 +0200 Subject: [PATCH 04/14] =?UTF-8?q?Print=20every=20line,=20don=E2=80=99t=20j?= =?UTF-8?q?oin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Used Enum.map to trigger IO.puts instead of joining and printing in a single blob. --- lib/onigumo/parser.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/onigumo/parser.ex b/lib/onigumo/parser.ex index 7008404..7d8a32c 100644 --- a/lib/onigumo/parser.ex +++ b/lib/onigumo/parser.ex @@ -7,7 +7,6 @@ defmodule Onigumo.Parser do root_path |> File.ls!() |> Enum.filter(fn filename -> Path.extname(filename) == ".raw" end) - |> Enum.join("\n") - |> IO.puts() + |> Enum.map(&IO.puts(&1)) end end From 8cf202eabb3eb6a730d8daffbad8aba9711a4ef9 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sat, 20 May 2023 19:50:43 +0200 Subject: [PATCH 05/14] Extract file listing to methods --- lib/onigumo/parser.ex | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/onigumo/parser.ex b/lib/onigumo/parser.ex index 7d8a32c..7fd7fb5 100644 --- a/lib/onigumo/parser.ex +++ b/lib/onigumo/parser.ex @@ -5,8 +5,18 @@ defmodule Onigumo.Parser do def main(root_path) do root_path - |> File.ls!() - |> Enum.filter(fn filename -> Path.extname(filename) == ".raw" end) + |> list_downloaded() |> Enum.map(&IO.puts(&1)) end + + defp list_downloaded(path) do + path + |> File.ls!() + |> Enum.filter(&is_downloaded(&1)) + end + + defp is_downloaded(path) do + suffix = Application.get_env(:onigumo, :downloaded_suffix) + Path.extname(path) == ".#{suffix}" + end end From 7223f367c33b019bb7aafcfc527a29f5084fe0ec Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sat, 21 Oct 2023 20:01:17 +0200 Subject: [PATCH 06/14] Fix indentation Replace tabs with spaces. --- lib/onigumo/parser.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/onigumo/parser.ex b/lib/onigumo/parser.ex index 7fd7fb5..7942e57 100644 --- a/lib/onigumo/parser.ex +++ b/lib/onigumo/parser.ex @@ -5,14 +5,14 @@ defmodule Onigumo.Parser do def main(root_path) do root_path - |> list_downloaded() + |> list_downloaded() |> Enum.map(&IO.puts(&1)) end defp list_downloaded(path) do - path + path |> File.ls!() - |> Enum.filter(&is_downloaded(&1)) + |> Enum.filter(&is_downloaded(&1)) end defp is_downloaded(path) do From 69a1305148d7c7a5b47043ec40c52b7983c3cd42 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Mon, 20 May 2024 09:49:52 +0200 Subject: [PATCH 07/14] Create a parser test stub MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently it doesn’t test anything for real. It’s just a test file with a single test to be filled in. --- test/onigumo_parser_test.exs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 test/onigumo_parser_test.exs diff --git a/test/onigumo_parser_test.exs b/test/onigumo_parser_test.exs new file mode 100644 index 0000000..f2a9bd9 --- /dev/null +++ b/test/onigumo_parser_test.exs @@ -0,0 +1,14 @@ +defmodule OnigumoParserTest do + use ExUnit.Case + import Mox + + setup(:verify_on_exit!) + + describe("Onigumo.Parser.main/1") do + @tag :tmp_dir + test("run Parser", %{tmp_dir: tmp_dir}) do + result = Onigumo.Parser.main(tmp_dir) + assert(result) + end + end +end From 8895452a58e498c768f511c6bdc73c2e8a990993 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sat, 25 May 2024 17:13:33 +0200 Subject: [PATCH 08/14] Add main result test The Onigumo.Component behavior requires main to return :ok. --- test/onigumo_parser_test.exs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/onigumo_parser_test.exs b/test/onigumo_parser_test.exs index f2a9bd9..5db22b1 100644 --- a/test/onigumo_parser_test.exs +++ b/test/onigumo_parser_test.exs @@ -7,8 +7,14 @@ defmodule OnigumoParserTest do describe("Onigumo.Parser.main/1") do @tag :tmp_dir test("run Parser", %{tmp_dir: tmp_dir}) do + _result = Onigumo.Parser.main(tmp_dir) + assert(false) # TODO: Do! + end + + @tag :tmp_dir + test("return :ok", %{tmp_dir: tmp_dir}) do result = Onigumo.Parser.main(tmp_dir) - assert(result) + assert(result == :ok) end end end From c20c6d2b89bb56312802fcb42e6889aee1010313 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sat, 25 May 2024 17:30:49 +0200 Subject: [PATCH 09/14] Test is_downloaded Made the function public, so it is testable. --- lib/onigumo/parser.ex | 2 +- test/onigumo_parser_test.exs | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/onigumo/parser.ex b/lib/onigumo/parser.ex index 1939ad5..d09bb21 100644 --- a/lib/onigumo/parser.ex +++ b/lib/onigumo/parser.ex @@ -19,7 +19,7 @@ defmodule Onigumo.Parser do |> Enum.filter(&is_downloaded(&1)) end - defp is_downloaded(path) do + def is_downloaded(path) do suffix = Application.get_env(:onigumo, :downloaded_suffix) Path.extname(path) == suffix end diff --git a/test/onigumo_parser_test.exs b/test/onigumo_parser_test.exs index 5db22b1..594f713 100644 --- a/test/onigumo_parser_test.exs +++ b/test/onigumo_parser_test.exs @@ -2,6 +2,11 @@ defmodule OnigumoParserTest do use ExUnit.Case import Mox + @suffixes [ + ".json", + "" + ] + setup(:verify_on_exit!) describe("Onigumo.Parser.main/1") do @@ -17,4 +22,23 @@ defmodule OnigumoParserTest do assert(result == :ok) end end + + describe("Onigumo.Parser.is_downloaded/1") do + test("recognize a downloaded file") do + suffix = Application.get_env(:onigumo, :downloaded_suffix) + path = "/test_dir/test_file#{suffix}" + + result = Onigumo.Parser.is_downloaded(path) + assert(result == true) + end + + for suffix <- @suffixes do + test("do not recognize another file #{inspect(suffix)}") do + path = "/test_dir/test_file#{unquote(suffix)}" + + result = Onigumo.Parser.is_downloaded(path) + assert(result == false) + end + end + end end From c73275fc359a51fc5ca38449e4cdf5eeaaf7acb9 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sat, 25 May 2024 17:40:03 +0200 Subject: [PATCH 10/14] First test of list_downloaded Test an empty directory. Made list_downloaded public for it to be testable. --- lib/onigumo/parser.ex | 2 +- test/onigumo_parser_test.exs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/onigumo/parser.ex b/lib/onigumo/parser.ex index d09bb21..9691253 100644 --- a/lib/onigumo/parser.ex +++ b/lib/onigumo/parser.ex @@ -13,7 +13,7 @@ defmodule Onigumo.Parser do :ok end - defp list_downloaded(path) do + def list_downloaded(path) do path |> File.ls!() |> Enum.filter(&is_downloaded(&1)) diff --git a/test/onigumo_parser_test.exs b/test/onigumo_parser_test.exs index 594f713..60d4885 100644 --- a/test/onigumo_parser_test.exs +++ b/test/onigumo_parser_test.exs @@ -23,6 +23,14 @@ defmodule OnigumoParserTest do end end + describe("Onigumo.Parser.list_downloaded/1") do + @tag :tmp_dir + test("list an empty directory", %{tmp_dir: tmp_dir}) do + result = Onigumo.Parser.list_downloaded(tmp_dir) + assert(result = []) + end + end + describe("Onigumo.Parser.is_downloaded/1") do test("recognize a downloaded file") do suffix = Application.get_env(:onigumo, :downloaded_suffix) From bd812f876fd45f6a4f80f4d7f05c65d2f2b27141 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sun, 2 Jun 2024 17:22:32 +0200 Subject: [PATCH 11/14] Fix comparison Values are compared with ==, not with =. Fixing an unused variable warning. --- test/onigumo_parser_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/onigumo_parser_test.exs b/test/onigumo_parser_test.exs index 60d4885..4037eb9 100644 --- a/test/onigumo_parser_test.exs +++ b/test/onigumo_parser_test.exs @@ -27,7 +27,7 @@ defmodule OnigumoParserTest do @tag :tmp_dir test("list an empty directory", %{tmp_dir: tmp_dir}) do result = Onigumo.Parser.list_downloaded(tmp_dir) - assert(result = []) + assert(result == []) end end From 452430d01287a52df583350165b5890b91e1d2ba Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sun, 2 Jun 2024 17:44:24 +0200 Subject: [PATCH 12/14] Add a non-empty list_downloaded test --- test/onigumo_parser_test.exs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/onigumo_parser_test.exs b/test/onigumo_parser_test.exs index 4037eb9..587f88c 100644 --- a/test/onigumo_parser_test.exs +++ b/test/onigumo_parser_test.exs @@ -29,6 +29,18 @@ defmodule OnigumoParserTest do result = Onigumo.Parser.list_downloaded(tmp_dir) assert(result == []) end + + @tag :tmp_dir + test("list a non-empty directory", %{tmp_dir: tmp_dir}) do + suffix = Application.get_env(:onigumo, :downloaded_suffix) + files = ["first#{suffix}", "second#{suffix}"] + Enum.map(files, fn file -> + path = Path.join(tmp_dir, file) + File.write!(path, "") + end) + result = Onigumo.Parser.list_downloaded(tmp_dir) + assert(result == files) + end end describe("Onigumo.Parser.is_downloaded/1") do From 35cc96a14515a6b798fb811789f4d404eb435692 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sun, 2 Jun 2024 18:31:52 +0200 Subject: [PATCH 13/14] Compare ls output MapSets instead of Lists --- test/onigumo_parser_test.exs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/test/onigumo_parser_test.exs b/test/onigumo_parser_test.exs index 587f88c..16f6637 100644 --- a/test/onigumo_parser_test.exs +++ b/test/onigumo_parser_test.exs @@ -30,16 +30,21 @@ defmodule OnigumoParserTest do assert(result == []) end - @tag :tmp_dir - test("list a non-empty directory", %{tmp_dir: tmp_dir}) do - suffix = Application.get_env(:onigumo, :downloaded_suffix) - files = ["first#{suffix}", "second#{suffix}"] - Enum.map(files, fn file -> - path = Path.join(tmp_dir, file) - File.write!(path, "") - end) - result = Onigumo.Parser.list_downloaded(tmp_dir) - assert(result == files) + for {files, indices} <- @files do + @tag :tmp_dir + test("list a directory with #{inspect(files)} expecting #{inspect(indices)}", %{tmp_dir: tmp_dir}) do + suffix = Application.get_env(:onigumo, :downloaded_suffix) + files = MapSet.new(["first#{suffix}", "second#{suffix}"]) + Enum.map(files, fn file -> + path = Path.join(tmp_dir, file) + File.write!(path, "") + end) + + result = + Onigumo.Parser.list_downloaded(tmp_dir) + |> MapSet.new() + assert(result == files) + end end end From b61242badf92b2c133cdd9a6000f773ff65fe781 Mon Sep 17 00:00:00 2001 From: Glutexo Date: Sun, 2 Jun 2024 19:49:57 +0200 Subject: [PATCH 14/14] Test list_downloaded The parametrization allows to test various combinations of expected and unexpected files. --- test/onigumo_parser_test.exs | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/test/onigumo_parser_test.exs b/test/onigumo_parser_test.exs index 16f6637..c225f92 100644 --- a/test/onigumo_parser_test.exs +++ b/test/onigumo_parser_test.exs @@ -7,6 +7,13 @@ defmodule OnigumoParserTest do "" ] + @files [ + {[], []}, + {["first", "second"], []}, + {["first"], ["second"]}, + {[], ["first", "second"]}, + ] + setup(:verify_on_exit!) describe("Onigumo.Parser.main/1") do @@ -24,26 +31,22 @@ defmodule OnigumoParserTest do end describe("Onigumo.Parser.list_downloaded/1") do - @tag :tmp_dir - test("list an empty directory", %{tmp_dir: tmp_dir}) do - result = Onigumo.Parser.list_downloaded(tmp_dir) - assert(result == []) - end - - for {files, indices} <- @files do + for {expected, unexpected} <- @files do @tag :tmp_dir - test("list a directory with #{inspect(files)} expecting #{inspect(indices)}", %{tmp_dir: tmp_dir}) do + test("list a directory expecting #{inspect(expected)} and not expecting #{inspect(unexpected)}", %{tmp_dir: tmp_dir}) do suffix = Application.get_env(:onigumo, :downloaded_suffix) - files = MapSet.new(["first#{suffix}", "second#{suffix}"]) - Enum.map(files, fn file -> + + expected = Enum.map(unquote(expected), fn file -> + file <> suffix + end) + + Enum.map(expected ++ unquote(unexpected), fn file -> path = Path.join(tmp_dir, file) File.write!(path, "") end) - result = - Onigumo.Parser.list_downloaded(tmp_dir) - |> MapSet.new() - assert(result == files) + result = Onigumo.Parser.list_downloaded(tmp_dir) + assert(MapSet.new(result) == MapSet.new(expected)) end end end