Refactor paginator (#425)

* capsule some of the request calls

* final touches

* make mypy happy

* add missing file

* implement the new paginator design

* adjust tests to the new design

* trunk catches

* remove wrong file

* small improvements

* make mypy happy

* adjust expected result for doctest

* further smaller fixes

* fix vocab test

* automate cancelling outdated workflow runs

* fix doctest

* fix test expectations

* remove tests that don't really make sense. I don't think we should process user supplied URLs too much.

* fix trunk check

* make API and data schema smarter. Delay init of DBschema to raise API connection error at right place.

* full circle and we still don't have the correct search results

* reenable tests

* stupid mistakes

* remove unmerged paths

* add missing page increase

* add good changes

* less error prone

* raw is better for debugging

* fix import error

* remove obsolete test

.github/workflows/docs_check.yaml

@@ -15,6 +15,13 @@ on:
  - main
  - develop
  - "*"
+concurrency:
+ # github.workflow: name of the workflow
+ # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+ # Cancel in-progress runs when a new workflow with the same group name is triggered
+ cancel-in-progress: true
 
 jobs:
  build:

.github/workflows/doctest.yaml

@@ -14,6 +14,14 @@ on:
  - main
  - develop
 
+concurrency:
+ # github.workflow: name of the workflow
+ # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+ # Cancel in-progress runs when a new workflow with the same group name is triggered
+ cancel-in-progress: true
+
 jobs:
  doctest:
  strategy:

.github/workflows/mypy.yaml

@@ -14,6 +14,14 @@ on:
  - main
  - develop
 
+concurrency:
+ # github.workflow: name of the workflow
+ # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+ # Cancel in-progress runs when a new workflow with the same group name is triggered
+ cancel-in-progress: true
+
 jobs:
  mypy-test:
  strategy:

.github/workflows/test_coverage.yaml

@@ -15,6 +15,14 @@ on:
  - main
  - develop
 
+concurrency:
+ # github.workflow: name of the workflow
+ # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+ # Cancel in-progress runs when a new workflow with the same group name is triggered
+ cancel-in-progress: true
+
 jobs:
  test-coverage:
  runs-on: ubuntu-latest

.github/workflows/test_examples.yml

@@ -12,6 +12,14 @@ on:
  - main
  - develop
 
+concurrency:
+ # github.workflow: name of the workflow
+ # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+ # Cancel in-progress runs when a new workflow with the same group name is triggered
+ cancel-in-progress: true
+
 jobs:
  test-examples:
  runs-on: ${{ matrix.os }}

.github/workflows/tests.yml

@@ -18,6 +18,14 @@ on:
  - develop
  - "*"
 
+concurrency:
+ # github.workflow: name of the workflow
+ # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+ # Cancel in-progress runs when a new workflow with the same group name is triggered
+ cancel-in-progress: true
+
 jobs:
  install:
  runs-on: ${{ matrix.os }}

conftest.py

@@ -8,6 +8,7 @@
 
 The fixtures are all functional fixtures that stay consistent between all tests.
 """
+import logging
 import os
 
 import pytest
@@ -49,14 +50,15 @@ def cript_api():
  """
  storage_token = os.getenv("CRIPT_STORAGE_TOKEN")
 
- with cript.API(host=None, api_token=None, storage_token=storage_token) as api:
+ with cript.API(host=None, api_token=None, storage_token=storage_token, default_log_level=logging.DEBUG) as api:
  # overriding AWS S3 cognito variables to be sure we do not upload test data to production storage
  # staging AWS S3 cognito storage variables
  api._IDENTITY_POOL_ID = "us-east-1:25043452-a922-43af-b8a6-7e938a9e55c1"
  api._COGNITO_LOGIN_PROVIDER = "cognito-idp.us-east-1.amazonaws.com/us-east-1_vyK1N9p22"
  api._BUCKET_NAME = "cript-stage-user-data"
  # using the tests folder name within our cloud storage
  api._BUCKET_DIRECTORY_NAME = "tests"
+ api.extra_api_log_debug_info = True
 
  yield api