Housekeeping (#90)

* initial commit * Update adapter.py * new: controller example in README * new: `track_adb`, `track_nx` * Update utils.py * fix: isort * cleanup: progress tracker * reignite pr readme badge fix, try for 3.11 support, lock lint/formatter dependency versions * remove: python 3.7 support (eol) * temp: remove mypy for tests * cleanup * checkpoint * fix: test for batch size (adb to nx) * fix: black * update notebook * fix: `process_adb_cursor` * fix: `abc` * fix: `nx_graph`, typos * cleanup * cleanup: `rich` * update notebooks * fix docstring * update readme & notebooks * more cleanup * cleanup * cleanup: `examples` directory * cleanup workflows * Update build.yml * Update build.yml * update release action * Update README.md * Update README.md * Update README.md * migrate to `pyproject.toml` * Update build.yml * fix lint * fix lint * disable python 3.12 * re-enable 3.12 * flake8 extend ignore trying to workaround 3.12 builds: https://github.com/arangoml/networkx-adapter/actions/runs/6857027442/job/18645348203?pr=90 * optimize: `__process_adb_vertex` * fix var name * use `prepare_adb_vertex_method_is_empty` instead of `is_custom_controller` * remove breakpoint
arangoml · Nov 22, 2023 · 8b83c7d · 8b83c7d
1 parent ca20f7e
commit 8b83c7d
Show file tree

Hide file tree

Showing 53 changed files with 11,481 additions and 403,568 deletions.
diff --git a/.github/workflows/analyze.yml b/.github/workflows/analyze.yml
@@ -37,7 +37,7 @@ jobs:
 
  steps:
  - name: Checkout repository
- uses: actions/checkout@v2
+ uses: actions/checkout@v4
 
  # Initializes the CodeQL tools for scanning.
  - name: Initialize CodeQL

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -1,9 +1,8 @@
 name: build
 on:
  workflow_dispatch:
- push:
- branches: [ master ]
  pull_request:
+ push:
  branches: [ master ]
 env:
  PACKAGE_DIR: adbnx_adapter
@@ -13,34 +12,47 @@ jobs:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- python: ["3.7", "3.8", "3.9", "3.10"]
+ python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
  name: Python ${{ matrix.python }}
  steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v4
+
  - name: Setup Python ${{ matrix.python }}
- uses: actions/setup-python@v2
+ uses: actions/setup-python@v4
  with:
  python-version: ${{ matrix.python }}
+ cache: 'pip'
+ cache-dependency-path: setup.py
+
  - name: Set up ArangoDB Instance via Docker
- run: docker create --name adb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb:3.9.1
+ run: docker create --name adb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
+
  - name: Start ArangoDB Instance
  run: docker start adb
+
  - name: Setup pip
  run: python -m pip install --upgrade pip setuptools wheel
+
  - name: Install packages
  run: pip install .[dev]
+
  - name: Run black
  run: black --check --verbose --diff --color ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
+
  - name: Run flake8
  run: flake8 ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
+
  - name: Run isort
  run: isort --check --profile=black ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
+
  - name: Run mypy
  run: mypy ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
+
  - name: Run pytest
  run: pytest --cov=${{env.PACKAGE_DIR}} --cov-report xml --cov-report term-missing -v --color=yes --no-cov-on-fail --code-highlight=yes
+
  - name: Publish to coveralls.io
- if: matrix.python == '3.8'
+ if: matrix.python == '3.10'
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  run: coveralls --service=github
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -3,76 +3,34 @@ on:
  workflow_dispatch:
  release:
  types: [published]
-env:
- PACKAGE_DIR: adbnx_adapter
- TESTS_DIR: tests
 jobs:
- build:
- runs-on: ubuntu-latest
- strategy:
- matrix:
- python: ["3.7", "3.8", "3.9", "3.10"]
- name: Python ${{ matrix.python }}
- steps:
- - uses: actions/checkout@v2
- - name: Setup Python ${{ matrix.python }}
- uses: actions/setup-python@v2
- with:
- python-version: ${{ matrix.python }}
- - name: Set up ArangoDB Instance via Docker
- run: docker create --name adb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb:3.9.1
- - name: Start ArangoDB Instance
- run: docker start adb
- - name: Setup pip
- run: python -m pip install --upgrade pip setuptools wheel
- - name: Install packages
- run: pip install .[dev]
- - name: Run black
- run: black --check --verbose --diff --color ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
- - name: Run flake8
- run: flake8 ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
- - name: Run isort
- run: isort --check --profile=black ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
- - name: Run mypy
- run: mypy ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}
- - name: Run pytest
- run: pytest --cov=${{env.PACKAGE_DIR}} --cov-report xml --cov-report term-missing -v --color=yes --no-cov-on-fail --code-highlight=yes
- - name: Publish to coveralls.io
- if: matrix.python == '3.8'
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- run: coveralls --service=github
-
  release:
- needs: build
  runs-on: ubuntu-latest
  name: Release package
  steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v4
 
  - name: Fetch complete history for all tags and branches
  run: git fetch --prune --unshallow
 
- - name: Setup python
- uses: actions/setup-python@v2
+ - name: Setup Python
+ uses: actions/setup-python@v4
  with:
- python-version: "3.8"
+ python-version: "3.10"
 
  - name: Install release packages
  run: pip install setuptools wheel twine setuptools-scm[toml]
 
- - name: Install dependencies
- run: pip install .[dev]
-
  - name: Build distribution
  run: python setup.py sdist bdist_wheel
 
- - name: Publish to PyPI Test
+ - name: Publish to Test PyPi
  env:
  TWINE_USERNAME: __token__
  TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD_TEST }}
  run: twine upload --repository testpypi dist/* #--skip-existing
- - name: Publish to PyPI
+
+ - name: Publish to PyPi
  env:
  TWINE_USERNAME: __token__
  TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
@@ -83,7 +41,7 @@ jobs:
  runs-on: ubuntu-latest
  name: Update Changelog
  steps:
- - uses: actions/checkout@v2
+ - uses: actions/checkout@v4
  with:
  fetch-depth: 0
 
@@ -95,10 +53,10 @@ jobs:
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
- - name: Setup python
- uses: actions/setup-python@v2
+ - name: Setup Python
+ uses: actions/setup-python@v4
  with:
- python-version: "3.8"
+ python-version: "3.10"
 
  - name: Install release packages
  run: pip install wheel gitchangelog pystache
@@ -110,12 +68,12 @@ jobs:
  run: gitchangelog ${{env.VERSION}} > CHANGELOG.md
 
  - name: Make commit for auto-generated changelog
- uses: EndBug/add-and-commit@v7
+ uses: EndBug/add-and-commit@v9
  env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  with:
  add: "CHANGELOG.md"
- branch: actions/changelog
+ new_branch: actions/changelog
  message: "!gitchangelog"
 
  - name: Create pull request for the auto generated changelog

diff --git a/README.md b/README.md
@@ -9,7 +9,7 @@
 
 [![License](https://img.shields.io/github/license/arangoml/networkx-adapter?color=9E2165&style=for-the-badge)](https://github.com/arangoml/networkx-adapter/blob/master/LICENSE)
 [![Code style: black](https://img.shields.io/static/v1?style=for-the-badge&label=code%20style&message=black&color=black)](https://github.com/psf/black)
-[![Downloads](https://img.shields.io/badge/dynamic/json?style=for-the-badge&color=282661&label=Downloads&query=total_downloads&url=https://api.pepy.tech/api/projects/adbnx-adapter)](https://pepy.tech/project/adbnx-adapter)
+[![Downloads](https://img.shields.io/badge/dynamic/json?style=for-the-badge&color=282661&label=Downloads&query=total_downloads&url=https://api.pepy.tech/api/v2/projects/adbnx-adapter)](https://pepy.tech/project/adbnx-adapter)
 
 <a href="https://www.arangodb.com/" rel="arangodb.com">![](https://raw.githubusercontent.com/arangoml/networkx-adapter/master/examples/assets/logos/ArangoDB_logo.png)</a>
 <a href="https://networkx.org/" rel="networkx.org">![](https://raw.githubusercontent.com/arangoml/networkx-adapter/master/examples/assets/logos/networkx_logo.svg)</a>
@@ -45,30 +45,40 @@ pip install git+https://github.com/arangoml/networkx-adapter.git
 Also available as an ArangoDB Lunch & Learn session: [Graph & Beyond Course #2.9](https://www.arangodb.com/resources/lunch-sessions/graph-beyond-lunch-break-2-9-introducing-the-arangodb-networkx-adapter/)
 
 ```py
-from arango import ArangoClient # Python-Arango driver
 import networkx as nx
 
+from arango import ArangoClient
 from adbnx_adapter import ADBNX_Adapter, ADBNX_Controller
 
-# Let's assume that the ArangoDB "fraud detection" dataset is imported to this endpoint
-db = ArangoClient(hosts="http://localhost:8529").db("_system", username="root", password="")
+# Connect to ArangoDB
+db = ArangoClient().db()
 
+# Instantiate the adapter
 adbnx_adapter = ADBNX_Adapter(db)
 ```
 
 ### ArangoDB to NetworkX
 ```py
-# 1.1: ArangoDB to NetworkX via Graph name
+#######################
+# 1.1: via Graph name #
+#######################
+
 nx_g = adbnx_adapter.arangodb_graph_to_networkx("fraud-detection")
 
-# 1.2: ArangoDB to NetworkX via Collection names
+#############################
+# 1.2: via Collection names #
+#############################
+
 nx_g = adbnx_adapter.arangodb_collections_to_networkx(
  "fraud-detection", 
  {"account", "bank", "branch", "Class", "customer"}, # Vertex collections
  {"accountHolder", "Relationship", "transaction"} # Edge collections
 )
 
-# 1.3: ArangoDB to NetworkX via Metagraph
+######################
+# 1.3: via Metagraph #
+######################
+
 metagraph = {
  "vertexCollections": {
  "account": {"Balance", "account_type", "customer_id", "rank"},
@@ -79,10 +89,13 @@ metagraph = {
  "accountHolder": {},
  },
 }
+
 nx_g = adbnx_adapter.arangodb_to_networkx("fraud-detection", metagraph)
 
+#######################################
+# 1.4: with a custom ADBNX Controller #
+#######################################
 
-# 1.4: ArangoDB to NetworkX with a custom ADBNX Controller
 class Custom_ADBNX_Controller(ADBNX_Controller):
  """ArangoDB-NetworkX controller.
 
@@ -113,7 +126,10 @@ nx_g = ADBNX_Adapter(db, Custom_ADBNX_Controller()).arangodb_graph_to_networkx("
 
 ### NetworkX to ArangoDB
 ```py
-# 2.1: NetworkX Homogeneous graph to ArangoDB
+#################################
+# 2.1: with a Homogeneous Graph #
+#################################
+
 nx_g = nx.grid_2d_graph(5, 5)
 edge_definitions = [
  {
@@ -122,29 +138,44 @@ edge_definitions = [
  "to_vertex_collections": ["Grid_Node"],
  }
 ]
+
 adb_g = adbnx_adapter.networkx_to_arangodb("Grid", nx_g, edge_definitions)
 
-# 2.2: NetworkX Heterogeneous graph to ArangoDB with ArangoDB node IDs
-edges = []
-for i in range(1, 101):
- for j in range(1, 101):
- if j % i == 0:
- # Notice that the NetworkX node IDs are following ArangoDB _id formatting standards (i.e `collection_name/node_key`)
- edges.append((f"numbers_j/{j}", f"numbers_i/{i}", j / i)) 
+#############################################################
+# 2.2: with a Homogeneous Graph & a custom ADBNX Controller #
+#############################################################
 
-nx_g = nx.MultiDiGraph()
-nx_g.add_weighted_edges_from(edges)
+class Custom_ADBNX_Controller(ADBNX_Controller):
+ """ArangoDB-NetworkX controller.
 
-edge_definitions = [
- {
- "edge_collection": "is_divisible_by",
- "from_vertex_collections": ["numbers_j"],
- "to_vertex_collections": ["numbers_i"],
- }
-]
-adb_g = adbnx_adapter.networkx_to_arangodb("Divisibility", nx_g, edge_definitions, keyify_nodes=True)
+ Responsible for controlling how nodes & edges are handled when
+ transitioning from ArangoDB to NetworkX, and vice-versa.
+ """
+
+ def _prepare_networkx_node(self, nx_node: dict, col: str) -> None:
+ """Prepare a NetworkX node before it gets inserted into the ArangoDB
+ collection **col**.
+
+ :param nx_node: The NetworkX node object to (optionally) modify.
+ :param col: The ArangoDB collection the node belongs to.
+ """
+ nx_node["foo"] = "bar"
+
+ def _prepare_networkx_edge(self, nx_edge: dict, col: str) -> None:
+ """Prepare a NetworkX edge before it gets inserted into the ArangoDB
+ collection **col**.
+
+ :param nx_edge: The NetworkX edge object to (optionally) modify.
+ :param col: The ArangoDB collection the edge belongs to.
+ """
+ nx_edge["bar"] = "foo"
+
+adb_g = ADBNX_Adapter(db, Custom_ADBNX_Controller()).networkx_to_arangodb("Grid", nx_g, edge_definitions)
+
+###################################
+# 2.3: with a Heterogeneous Graph #
+###################################
 
-# 2.3 NetworkX Heterogeneous graph to ArangoDB with non-ArangoDB node IDs
 edges = [
  ('student:101', 'lecture:101'), 
  ('student:102', 'lecture:102'), 
@@ -160,7 +191,10 @@ edges = [
 nx_g = nx.MultiDiGraph()
 nx_g.add_edges_from(edges)
 
-### Learn how this example is handled in Colab: https://colab.research.google.com/github/arangoml/networkx-adapter/blob/master/examples/ArangoDB_NetworkX_Adapter.ipynb#scrollTo=OuU0J7p1E9OM
+# ...
+
+# Learn how this example is handled in Colab:
+# https://colab.research.google.com/github/arangoml/networkx-adapter/blob/master/examples/ArangoDB_NetworkX_Adapter.ipynb#scrollTo=OuU0J7p1E9OM
 ```
 
 ## Development & Testing

diff --git a/adbnx_adapter/__init__.py b/adbnx_adapter/__init__.py
@@ -1,2 +1,5 @@
 from adbnx_adapter.adapter import ADBNX_Adapter # noqa: F401
-from adbnx_adapter.controller import ADBNX_Controller # noqa: F401
+from adbnx_adapter.controller import ( # noqa: F401
+ ADBNX_Controller,
+ ADBNX_Controller_Full_Cycle,
+)