Merge pull request #8 from soramitsu/filter-retry-cache

feat: filter by `deleted`, retry overview, update cache policy

Dockerfile

@@ -1,4 +1,4 @@
-FROM denoland/deno:alpine-1.35.2
+FROM denoland/deno:alpine-1.45.3
 WORKDIR /app
 COPY deps.ts deno.lock ./
 RUN deno cache --lock=deno.lock deps.ts

README.md

@@ -52,3 +52,10 @@ If needed remove deno.lock first
 ```bash
 deno cache --lock=deno.lock deps.ts
 ```
+
+## Prototyping
+
+There is [`allure_compat_matrix.livemd`](./allure_compat_matrix.livemd) - a
+[Livebook](https://livebook.dev/) notebook used for prototyping of the data
+flow. It can be used as a reference and as a tool for further updates. Please
+refer to the Livebook documentation on how to use it if you want.

allure_compat_matrix.livemd

@@ -0,0 +1,157 @@
+# Allure, Compat Matrix
+
+```elixir
+Mix.install([
+  {:req, "~> 0.5.4"},
+  {:kino, "~> 0.13.2"}
+])
+```
+
+## Buildin the matrix
+
+Make sure to define `LB_ALLURE_API_TOKEN` in the environment.
+
+```elixir
+api_url = "https://soramitsu.testops.cloud"
+api_token = System.get_env("LB_ALLURE_API_TOKEN")
+
+create_date_after =
+  Kino.Input.utc_datetime("create date after (UTC)",
+    default: DateTime.utc_now() |> DateTime.add(-365, :day)
+  )
+```
+
+```elixir
+create_date_after =
+  Kino.Input.read(create_date_after) |> DateTime.from_naive!("Etc/UTC") |> DateTime.to_unix()
+
+headers = %{"Authorization" => "Api-Token #{api_token}"}
+
+response =
+  Req.get!("#{api_url}/api/rs/testresult/__search?",
+    params: [
+      projectId: 1,
+      rql: "not cf[\"SDK\"] is null and createdDate > #{create_date_after}",
+      page: 0,
+      size: 999_999,
+      sort: "created_date,DESC"
+    ],
+    headers: headers
+  )
+
+
+%{"content" => content} = response.body
+```
+
+```elixir
+require Logger
+
+meta =
+  content
+  |> Stream.map(& &1["testCaseId"])
+  |> Enum.into(MapSet.new())
+  |> Task.async_stream(fn id ->
+    response = Req.get!("#{api_url}/api/rs/testcase/#{id}/overview", headers: headers)
+    Logger.debug("Loaded overview of #{id}")
+
+    case response.body do
+      %{"deleted" => true} ->
+        Logger.debug("Test case #{id} is deleted, ignoring")
+        nil
+
+      %{"customFields" => custom_fields} ->
+        custom_fields =
+          custom_fields
+          |> Enum.map(fn %{"customField" => %{"id" => id, "name" => name}, "name" => value} ->
+            {name, %{id: id, value: value}}
+          end)
+          |> Enum.into(%{})
+
+        case custom_fields do
+          %{"SDK" => sdk, "Story" => story} ->
+            {id, %{sdk: sdk, story: story}}
+
+          bad ->
+            Logger.warning(
+              "test case #{id} doesn't have SDK or Story custom fields: #{inspect(bad)}"
+            )
+
+            nil
+        end
+
+      unknown ->
+        Logger.warning("response not recognised: #{inspect(unknown)}")
+        nil
+    end
+  end)
+  |> Stream.map(fn {:ok, data} -> data end)
+  |> Stream.reject(&(&1 == nil))
+  |> Enum.into(%{})
+```
+
+```elixir
+results_filtered =
+  content
+  |> Stream.map(&{&1["testCaseId"], &1})
+  |> Enum.dedup_by(&elem(&1, 0))
+  |> Enum.into(%{})
+```
+
+```elixir
+stories =
+  meta
+  |> Map.to_list()
+  |> Enum.reduce(%{}, fn {test_case_id, %{story: story, sdk: sdk}}, acc ->
+    result = Map.fetch!(results_filtered, test_case_id)
+
+    Map.update(acc, story.value, %{}, fn story_map ->
+      Map.update(story_map, sdk.value, [], fn list -> [result["status"] | list] end)
+    end)
+  end)
+```
+
+```elixir
+sdks =
+  stories
+  |> Map.values()
+  |> Stream.flat_map(&Map.keys/1)
+  |> Enum.into(MapSet.new())
+  |> MapSet.to_list()
+```
+
+```elixir
+matrix =
+  stories
+  |> Enum.map(fn {story, story_sdk_results} ->
+    results =
+      sdks
+      |> Stream.map(&Map.get(story_sdk_results, &1))
+      |> Enum.map(fn
+        nil -> :no_data
+        statuses -> if Enum.all?(statuses, &(&1 == "passed")), do: :ok, else: :failed
+      end)
+
+    %{name: story, results: results}
+  end)
+
+%{
+  sdks: sdks,
+  stories: matrix
+}
+```
+
+```elixir
+matrix
+|> Enum.map(fn %{name: name, results: results} ->
+  %{name: name}
+  |> Map.merge(
+    sdks
+    |> Enum.with_index()
+    |> Enum.map(fn {x, i} -> {x, Enum.at(results, i)} end)
+    |> Enum.into(%{})
+  )
+end)
+|> Kino.DataTable.new()
+```
+
+<!-- livebook:{"offset":3381,"stamp":{"token":"XCP.OpqmL3akQyxgIZygWtwE4L8hmcjPKzOmpfOtVjl_F9_oKhNADGXwbK_2rzudeW54BEIgGjhSrvxvZRazLoe6dDF7VdLPDrt1jLGQQywC44HE8bRkQYc2_20y5g","version":2}} -->

deps.ts

@@ -1,10 +1,10 @@
 export * as log from "https://deno.land/std@0.194.0/log/mod.ts";
 export { getLogger } from "https://deno.land/std@0.194.0/log/mod.ts";
-export { ms } from "https://deno.land/x/ms@v0.1.0/ms.ts";
 export * as oak from "https://deno.land/x/oak@v12.6.0/mod.ts";
 export { default as oakLogger } from "https://deno.land/x/oak_logger@1.0.0/mod.ts";
 export { match, P } from "https://esm.sh/ts-pattern@5.0.3";
 export { sortBy } from "https://deno.land/std@0.197.0/collections/mod.ts";
+export { retry } from "https://deno.land/x/retry@v2.0.0/mod.ts";
 
 export async function dotenvLoad() {
   await import("https://deno.land/std@0.194.0/dotenv/load.ts");

src/aggregate.ts

@@ -83,9 +83,12 @@ async function getTestCaseData(
 ): Promise<MetaMap> {
   const entries = await Promise.all(
     testCaseIdList.map(async (id): Promise<[number, TestCaseMeta][]> => {
-		await delay(50000);
-      const custom_fields = await api.getTestCaseCustomFields(id);
-      const map = customFieldsToMap(custom_fields);
+      const { deleted, customFields } = await api.getTestCaseOverview(id);
+      if (deleted) {
+        logger().debug({ msg: "ignore deleted test case", id });
+        return [];
+      }
+      const map = customFieldsToMap(customFields);
 
       if (!map.has(CUSTOM_FIELD_SDK)) {
         logger().warning({
@@ -121,26 +124,14 @@ async function getTestCaseData(
 function customFieldsToMap(
   input: ApiTestCaseCustomFieldData[],
 ): Map<string, CustomFieldData> {
-  if (!input) {
-    logger().warning({
-          msg: `Missing test!`
-        })
-    return new Map(null);
-  } else {
   const entries = input
     .map((x) => ({
       id: x.customField.id,
       name: x.customField.name,
       value: x.name,
     }))
     .map((x): [string, CustomFieldData] => [x.name, x]);
-    return new Map(entries);
-}
-
-}
-
-function delay(ms: number) {
-    return new Promise( resolve => setTimeout(resolve, ms) );
+  return new Map(entries);
 }
 
 function aggregateStories(
@@ -216,4 +207,4 @@ function pickResults<T extends ApiTestCaseResult>(
         map.has(item.testCaseId) ? map : map.set(item.testCaseId, item),
       new Map<number, T>(),
     );
-}
+}