diff --git a/Changelog b/Changelog index 3bea1704..442c9c9f 100644 --- a/Changelog +++ b/Changelog @@ -1,7 +1,10 @@ -Version 5.2.0 2023-11 +ersion 5.2.0 2023-11 +* Add an option to pass your own driver instead of relying on the automatically created one. See set_connection method. +* Add a close_connection method to explicitly close the driver to match Neo4j deprecation. +* Add a DATABASE_NAME config option, available for both auto- and self-managed driver modes. +* Add neomodel_inspect_database script, which inspects an existing database and creates neomodel class definitions for all objects. * Add support for pandas DataFrame and Series ; numpy Array * Add relationship uniqueness constraints - for Neo4j >= 5.7 -* Add neomodel_inspect_database script, which inspects an existing database and creates neomodel class definitions for all objects. Version 5.1.2 2023-09 * Raise ValueError on reserved keywords ; add tests #590 #623 diff --git a/README.rst b/README.rst index 1a66076c..c4edceb0 100644 --- a/README.rst +++ b/README.rst @@ -55,20 +55,6 @@ Documentation .. _readthedocs: http://neomodel.readthedocs.org -Upcoming breaking changes notice - >=5.2 -======================================== - -As part of the current quality improvement efforts, we are planning a rework of neomodel's main Database object, which will lead to breaking changes. - -The full scope is not drawn out yet, but here are the main points : - -* Refactoring standalone methods that depend on the Database singleton into the class itself. See issue https://github.com/neo4j-contrib/neomodel/issues/739 - -* Adding an option to pass your own driver to neomodel instead of relying on the one that the library creates for you. This will not be a breaking change. - -We are aiming to release this in neomodel 5.2 - - Installation ============ diff --git a/doc/source/conf.py b/doc/source/conf.py index 41e50bbf..538c5190 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -57,7 +57,7 @@ # General information about the project. project = __package__ -copyright = "2019, " + __author__ +copyright = "2023, " + __author__ # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/doc/source/configuration.rst b/doc/source/configuration.rst index 854af8a8..d5299f54 100644 --- a/doc/source/configuration.rst +++ b/doc/source/configuration.rst @@ -3,14 +3,24 @@ Configuration This section is covering the Neomodel 'config' module and its variables. -Database --------- +.. _connection_options_doc: -Setting the connection URL:: +Connection +---------- + +There are two ways to define your connection to the database : + +1. Provide a Neo4j URL and some options - Driver will be managed by neomodel +2. Create your own Neo4j driver and pass it to neomodel + +neomodel-managed (default) +-------------------------- + +Set the connection URL:: config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687` -Adjust driver configuration:: +Adjust driver configuration - these options are only available for this connection method:: config.MAX_CONNECTION_POOL_SIZE = 100 # default config.CONNECTION_ACQUISITION_TIMEOUT = 60.0 # default @@ -22,12 +32,90 @@ Adjust driver configuration:: config.MAX_TRANSACTION_RETRY_TIME = 30.0 # default config.RESOLVER = None # default config.TRUST = neo4j.TRUST_SYSTEM_CA_SIGNED_CERTIFICATES # default - config.USER_AGENT = neomodel/vNeo4j.Major.minor # default + config.USER_AGENT = neomodel/v5.2.0 # default -Setting the database name, for neo4j >= 4:: +Setting the database name, if different from the default one:: + # Using the URL only config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687/mydb` + # Using config option + config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687` + config.DATABASE_NAME = 'mydb' + +self-managed +------------ + +Create a Neo4j driver:: + + from neo4j import GraphDatabase + my_driver = GraphDatabase().driver('bolt://localhost:7687', auth=('neo4j', 'password')) + config.DRIVER = my_driver + +See the `driver documentation ` here. + +This mode allows you to use all the available driver options that neomodel doesn't implement, for example auth tokens for SSO. +Note that you have to manage the driver's lifecycle yourself. + +However, everything else is still handled by neomodel : sessions, transactions, etc... + +NB : Only the synchronous driver will work in this way. The asynchronous driver is not supported yet. + +Change/Close the connection +--------------------------- + +Optionally, you can change the connection at any time by calling ``set_connection``:: + + from neomodel import db + # Using URL - auto-managed + db.set_connection(url='bolt://neo4j:neo4j@localhost:7687') + + # Using self-managed driver + db.set_connection(driver=my_driver) + +The new connection url will be applied to the current thread or process. + +Since Neo4j version 5, driver auto-close is deprecated. Make sure to close the connection anytime you want to replace it, +as well as at the end of your application's lifecycle by calling ``close_connection``:: + + from neomodel import db + db.close_connection() + + # If you then want a new connection + db.set_connection(url=url) + +This will close the Neo4j driver, and clean up everything that neomodel creates for its internal workings. + +Protect your credentials +------------------------ + +You should `avoid setting database access credentials in plain sight `_. Neo4J defines a number of +`environment variables `_ that are used in its tools and these can be re-used for other applications +too. + +These are: + +* ``NEO4J_USERNAME`` +* ``NEO4J_PASSWORD`` +* ``NEO4J_BOLT_URL`` + +By setting these with (for example): :: + + $ export NEO4J_USERNAME=neo4j + $ export NEO4J_PASSWORD=neo4j + $ export NEO4J_BOLT_URL="bolt://$NEO4J_USERNAME:$NEO4J_PASSWORD@localhost:7687" + +They can be accessed from a Python script via the ``environ`` dict of module ``os`` and be used to set the connection +with something like: :: + + import os + from neomodel import config + + config.DATABASE_URL = os.environ["NEO4J_BOLT_URL"] + + Enable automatic index and constraint creation ---------------------------------------------- diff --git a/doc/source/getting_started.rst b/doc/source/getting_started.rst index d49630fb..4b832190 100644 --- a/doc/source/getting_started.rst +++ b/doc/source/getting_started.rst @@ -10,46 +10,13 @@ Before executing any neomodel code, set the connection url:: from neomodel import config config.DATABASE_URL = 'bolt://neo4j:neo4j@localhost:7687' # default - # You can specify a database name: 'bolt://neo4j:neo4j@localhost:7687/mydb' - This must be called early on in your app, if you are using Django the `settings.py` file is ideal. +See the Configuration page (:ref:`connection_options_doc`) for config options. + If you are using your neo4j server for the first time you will need to change the default password. This can be achieved by visiting the neo4j admin panel (default: ``http://localhost:7474`` ). -You can also change the connection url at any time by calling ``set_connection``:: - - from neomodel import db - db.set_connection('bolt://neo4j:neo4j@localhost:7687') - -The new connection url will be applied to the current thread or process. - -In general however, it is better to `avoid setting database access credentials in plain sight `_. Neo4J defines a number of -`environment variables `_ that are used in its tools and these can be re-used for other applications -too. - -These are: - -* ``NEO4J_USERNAME`` -* ``NEO4J_PASSWORD`` -* ``NEO4J_BOLT_URL`` - -By setting these with (for example): :: - - $ export NEO4J_USERNAME=neo4j - $ export NEO4J_PASSWORD=neo4j - $ export NEO4J_BOLT_URL="bolt://$NEO4J_USERNAME:$NEO4J_PASSWORD@localhost:7687" - -They can be accessed from a Python script via the ``environ`` dict of module ``os`` and be used to set the connection -with something like: :: - - import os - from neomodel import config - - config.DATABASE_URL = os.environ["NEO4J_BOLT_URL"] - Querying the graph ================== diff --git a/neomodel/config.py b/neomodel/config.py index 1f6df10b..b54aa806 100644 --- a/neomodel/config.py +++ b/neomodel/config.py @@ -3,6 +3,9 @@ from ._version import __version__ AUTO_INSTALL_LABELS = False + +# Use this to connect with automatically created driver +# The following options are the default ones that will be used as driver config DATABASE_URL = "bolt://neo4j:foobarbaz@localhost:7687" FORCE_TIMEZONE = False @@ -16,3 +19,8 @@ RESOLVER = None TRUSTED_CERTIFICATES = neo4j.TrustSystemCAs() USER_AGENT = f"neomodel/v{__version__}" + +# Use this to connect with your self-managed driver instead +# DRIVER = neo4j.GraphDatabase().driver( +# "bolt://localhost:7687", auth=("neo4j", "foobarbaz") +# ) diff --git a/neomodel/scripts/neomodel_install_labels.py b/neomodel/scripts/neomodel_install_labels.py index 444838b2..8bd5119f 100755 --- a/neomodel/scripts/neomodel_install_labels.py +++ b/neomodel/scripts/neomodel_install_labels.py @@ -27,8 +27,8 @@ from __future__ import print_function import sys -from argparse import ArgumentParser, RawDescriptionHelpFormatter import textwrap +from argparse import ArgumentParser, RawDescriptionHelpFormatter from importlib import import_module from os import environ, path @@ -70,14 +70,16 @@ def load_python_module_or_file(name): def main(): parser = ArgumentParser( formatter_class=RawDescriptionHelpFormatter, - description=textwrap.dedent(""" + description=textwrap.dedent( + """ Setup indexes and constraints on labels in Neo4j for your neomodel schema. If a connection URL is not specified, the tool will look up the environment variable NEO4J_BOLT_URL. If that environment variable is not set, the tool will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 """ - )) + ), + ) parser.add_argument( "apps", @@ -107,7 +109,7 @@ def main(): # Connect after to override any code in the module that may set the connection print(f"Connecting to {bolt_url}") - db.set_connection(bolt_url) + db.set_connection(url=bolt_url) install_all_labels() diff --git a/neomodel/scripts/neomodel_remove_labels.py b/neomodel/scripts/neomodel_remove_labels.py index 58a57cdd..1ad6cc34 100755 --- a/neomodel/scripts/neomodel_remove_labels.py +++ b/neomodel/scripts/neomodel_remove_labels.py @@ -23,8 +23,8 @@ """ from __future__ import print_function -from argparse import ArgumentParser, RawDescriptionHelpFormatter import textwrap +from argparse import ArgumentParser, RawDescriptionHelpFormatter from os import environ from .. import db, remove_all_labels @@ -32,15 +32,17 @@ def main(): parser = ArgumentParser( - formatter_class=RawDescriptionHelpFormatter, - description=textwrap.dedent(""" + formatter_class=RawDescriptionHelpFormatter, + description=textwrap.dedent( + """ Drop all indexes and constraints on labels from schema in Neo4j database. If a connection URL is not specified, the tool will look up the environment variable NEO4J_BOLT_URL. If that environment variable is not set, the tool will attempt to connect to the default URL bolt://neo4j:neo4j@localhost:7687 """ - )) + ), + ) parser.add_argument( "--db", @@ -59,7 +61,7 @@ def main(): # Connect after to override any code in the module that may set the connection print(f"Connecting to {bolt_url}") - db.set_connection(bolt_url) + db.set_connection(url=bolt_url) remove_all_labels() diff --git a/neomodel/util.py b/neomodel/util.py index 88420bd6..cf4230e3 100644 --- a/neomodel/util.py +++ b/neomodel/util.py @@ -7,7 +7,7 @@ from typing import Optional, Sequence from urllib.parse import quote, unquote, urlparse -from neo4j import DEFAULT_DATABASE, GraphDatabase, basic_auth +from neo4j import DEFAULT_DATABASE, Driver, GraphDatabase, basic_auth from neo4j.api import Bookmarks from neo4j.exceptions import ClientError, ServiceUnavailable, SessionExpired from neo4j.graph import Node, Path, Relationship @@ -33,8 +33,11 @@ def wrapper(self, *args, **kwargs): else: _db = self - if not _db.url: - _db.set_connection(config.DATABASE_URL) + if not _db.driver: + if hasattr(config, "DRIVER") and config.DRIVER: + _db.set_connection(driver=config.DRIVER) + elif config.DATABASE_URL: + _db.set_connection(url=config.DATABASE_URL) return func(self, *args, **kwargs) @@ -78,9 +81,46 @@ def __init__(self): self._database_edition = None self.impersonated_user = None - def set_connection(self, url): + def set_connection(self, url: str = None, driver: Driver = None): """ - Sets the connection URL to the address a Neo4j server is set up at + Sets the connection up and relevant internal. This can be done using a Neo4j URL or a driver instance. + + Args: + url (str): Optionally, Neo4j URL in the form protocol://username:password@hostname:port/dbname. + When provided, a Neo4j driver instance will be created by neomodel. + + driver (neo4j.Driver): Optionally, a pre-created driver instance. + When provided, neomodel will not create a driver instance but use this one instead. + """ + if driver: + self.driver = driver + if hasattr(config, "DATABASE_NAME") and config.DATABASE_NAME: + self._database_name = config.DATABASE_NAME + elif url: + self._parse_driver_from_url(url=url) + + self._pid = os.getpid() + self._active_transaction = None + # Set to default database if it hasn't been set before + if self._database_name is None: + self._database_name = DEFAULT_DATABASE + + # Getting the information about the database version requires a connection to the database + self._database_version = None + self._database_edition = None + self._update_database_version() + + def _parse_driver_from_url(self, url: str) -> None: + """Parse the driver information from the given URL and initialize the driver. + + Args: + url (str): The URL to parse. + + Raises: + ValueError: If the URL format is not as expected. + + Returns: + None - Sets the driver and database_name as class properties """ p_start = url.replace(":", "", 1).find(":") + 2 p_end = url.rfind("@") @@ -128,14 +168,23 @@ def set_connection(self, url): parsed_url.scheme + "://" + hostname, **options ) self.url = url - self._pid = os.getpid() - self._active_transaction = None - self._database_name = DEFAULT_DATABASE if database_name == "" else database_name + # The database name can be provided through the url or the config + if database_name == "": + if hasattr(config, "DATABASE_NAME") and config.DATABASE_NAME: + self._database_name = config.DATABASE_NAME + else: + self._database_name = database_name - # Getting the information about the database version requires a connection to the database + def close_connection(self): + """ + Closes the currently open driver. + The driver should always be closed at the end of the application's lifecyle. + """ self._database_version = None self._database_edition = None - self._update_database_version() + self._database_name = None + self.driver.close() + self.driver = None @property def database_version(self): @@ -358,7 +407,8 @@ def cypher_query( :type: dict :param handle_unique: Whether or not to raise UniqueProperty exception on Cypher's ConstraintValidation errors :type: bool - :param retry_on_session_expire: Whether or not to attempt the same query again if the transaction has expired + :param retry_on_session_expire: Whether or not to attempt the same query again if the transaction has expired. + If you use neomodel with your own driver, you must catch SessionExpired exceptions yourself and retry with a new driver instance. :type: bool :param resolve_objects: Whether to attempt to resolve the returned nodes to data model objects automatically :type: bool @@ -420,7 +470,7 @@ def _run_cypher_query( raise exc_info[1].with_traceback(exc_info[2]) except SessionExpired: if retry_on_session_expire: - self.set_connection(self.url) + self.set_connection(url=self.url) return self.cypher_query( query=query, params=params, diff --git a/test/conftest.py b/test/conftest.py index a7877ef0..7ec261d6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,6 +8,10 @@ from neomodel import clear_neo4j_database, config, db from neomodel.util import version_tag_to_integer +NEO4J_URL = os.environ.get("NEO4J_URL", "bolt://localhost:7687") +NEO4J_USERNAME = os.environ.get("NEO4J_USERNAME", "neo4j") +NEO4J_PASSWORD = os.environ.get("NEO4J_PASSWORD", "foobarbaz") + def pytest_addoption(parser): """ @@ -81,6 +85,11 @@ def pytest_sessionstart(session): db.cypher_query("GRANT IMPERSONATE (troygreene) ON DBMS TO admin") +@pytest.hookimpl +def pytest_unconfigure(): + db.close_connection() + + def check_and_skip_neo4j_least_version(required_least_neo4j_version, message): """ Checks if the NEO4J_VERSION is at least `required_least_neo4j_version` and skips a test if not. diff --git a/test/test_connection.py b/test/test_connection.py index 702a4122..fb3524bb 100644 --- a/test/test_connection.py +++ b/test/test_connection.py @@ -1,11 +1,13 @@ import os +import time import pytest +from neo4j import GraphDatabase from neo4j.debug import watch -from neomodel import config, db +from neomodel import StringProperty, StructuredNode, config, db -INITIAL_URL = db.url +from .conftest import NEO4J_PASSWORD, NEO4J_URL, NEO4J_USERNAME @pytest.fixture(autouse=True) @@ -13,8 +15,8 @@ def setup_teardown(): yield # Teardown actions after tests have run # Reconnect to initial URL for potential subsequent tests - db.driver.close() - db.set_connection(INITIAL_URL) + db.close_connection() + db.set_connection(url=config.DATABASE_URL) @pytest.fixture(autouse=True, scope="session") @@ -23,11 +25,102 @@ def neo4j_logging(): yield +def get_current_database_name() -> str: + """ + Fetches the name of the currently active database from the Neo4j database. + + Returns: + - str: The name of the current database. + """ + results, meta = db.cypher_query("CALL db.info") + results_as_dict = [dict(zip(meta, row)) for row in results] + + return results_as_dict[0]["name"] + + +class Pastry(StructuredNode): + name = StringProperty(unique_index=True) + + +def test_set_connection_driver_works(): + # Verify that current connection is up + assert Pastry(name="Chocolatine").save() + db.close_connection() + + # Test connection using a driver + db.set_connection( + driver=GraphDatabase().driver(NEO4J_URL, auth=(NEO4J_USERNAME, NEO4J_PASSWORD)) + ) + assert Pastry(name="Croissant").save() + + +def test_config_driver_works(): + # Verify that current connection is up + assert Pastry(name="Chausson aux pommes").save() + db.close_connection() + + # Test connection using a driver defined in config + driver = GraphDatabase().driver(NEO4J_URL, auth=(NEO4J_USERNAME, NEO4J_PASSWORD)) + + config.DRIVER = driver + assert Pastry(name="Grignette").save() + + # Clear config + # No need to close connection - pytest teardown will do it + config.DRIVER = None + + +@pytest.mark.skipif( + db.database_edition != "enterprise", + reason="Skipping test for community edition - no multi database in CE", +) +def test_connect_to_non_default_database(): + database_name = "pastries" + db.cypher_query(f"CREATE DATABASE {database_name} IF NOT EXISTS") + db.close_connection() + + # Set database name in url - for url init only + db.set_connection(url=f"{config.DATABASE_URL}/{database_name}") + assert get_current_database_name() == "pastries" + + db.close_connection() + + # Set database name in config - for both url and driver init + config.DATABASE_NAME = database_name + + # url init + db.set_connection(url=config.DATABASE_URL) + assert get_current_database_name() == "pastries" + + db.close_connection() + + # driver init + db.set_connection( + driver=GraphDatabase().driver(NEO4J_URL, auth=(NEO4J_USERNAME, NEO4J_PASSWORD)) + ) + assert get_current_database_name() == "pastries" + + # Clear config + # No need to close connection - pytest teardown will do it + config.DATABASE_NAME = None + + +@pytest.mark.parametrize( + "url", ["bolt://user:password", "http://user:password@localhost:7687"] +) +def test_wrong_url_format(url): + with pytest.raises( + ValueError, + match=rf"Expecting url format: bolt://user:password@localhost:7687 got {url}", + ): + db.set_connection(url=url) + + @pytest.mark.parametrize("protocol", ["neo4j+s", "neo4j+ssc", "bolt+s", "bolt+ssc"]) def test_connect_to_aura(protocol): cypher_return = "hello world" default_cypher_query = f"RETURN '{cypher_return}'" - db.driver.close() + db.close_connection() _set_connection(protocol=protocol) result, _ = db.cypher_query(default_cypher_query) @@ -41,17 +134,5 @@ def _set_connection(protocol): AURA_TEST_DB_PASSWORD = os.environ["AURA_TEST_DB_PASSWORD"] AURA_TEST_DB_HOSTNAME = os.environ["AURA_TEST_DB_HOSTNAME"] - config.DATABASE_URL = f"{protocol}://{AURA_TEST_DB_USER}:{AURA_TEST_DB_PASSWORD}@{AURA_TEST_DB_HOSTNAME}" - db.set_connection(config.DATABASE_URL) - - -@pytest.mark.parametrize( - "url", ["bolt://user:password", "http://user:password@localhost:7687"] -) -def test_wrong_url_format(url): - prev_url = db.url - with pytest.raises( - ValueError, - match=rf"Expecting url format: bolt://user:password@localhost:7687 got {url}", - ): - db.set_connection(url) + database_url = f"{protocol}://{AURA_TEST_DB_USER}:{AURA_TEST_DB_PASSWORD}@{AURA_TEST_DB_HOSTNAME}" + db.set_connection(url=database_url) diff --git a/test/test_database_management.py b/test/test_database_management.py index 4a09c3e3..2a2ece34 100644 --- a/test/test_database_management.py +++ b/test/test_database_management.py @@ -60,12 +60,18 @@ def test_change_password(): new_url = f"bolt://neo4j:{new_password}@localhost:7687" util.change_neo4j_password(db, "neo4j", new_password) + db.close_connection() - db.set_connection(new_url) + db.set_connection(url=new_url) + db.close_connection() with pytest.raises(AuthError): - db.set_connection(prev_url) + db.set_connection(url=prev_url) - db.set_connection(new_url) + db.close_connection() + + db.set_connection(url=new_url) util.change_neo4j_password(db, "neo4j", prev_password) - db.set_connection(prev_url) + db.close_connection() + + db.set_connection(url=prev_url) diff --git a/test/test_multiprocessing.py b/test/test_multiprocessing.py index 9ae4340b..fb00675d 100644 --- a/test/test_multiprocessing.py +++ b/test/test_multiprocessing.py @@ -18,3 +18,4 @@ def test_concurrency(): results = p.map(thing_create, range(50)) for returned, sent in results: assert returned == sent + db.close_connection() diff --git a/test/test_transactions.py b/test/test_transactions.py index 35e7c01f..0481e2a7 100644 --- a/test/test_transactions.py +++ b/test/test_transactions.py @@ -3,7 +3,14 @@ from neo4j.exceptions import ClientError, TransactionError from pytest import raises -from neomodel import StringProperty, StructuredNode, UniqueProperty, db, install_labels +from neomodel import ( + StringProperty, + StructuredNode, + UniqueProperty, + config, + db, + install_labels, +) class APerson(StructuredNode): @@ -80,9 +87,9 @@ def test_read_transaction(): people = APerson.nodes.all() assert people - with pytest.raises(TransactionError): + with raises(TransactionError): with db.read_transaction: - with pytest.raises(ClientError) as e: + with raises(ClientError) as e: APerson(name="Gina").save() assert e.value.code == "Neo.ClientError.Statement.AccessMode" @@ -97,25 +104,12 @@ def test_write_transaction(): def double_transaction(): db.begin() - with pytest.raises(SystemError, match=r"Transaction in progress"): + with raises(SystemError, match=r"Transaction in progress"): db.begin() db.rollback() -def test_set_connection_works(): - assert APerson(name="New guy 1").save() - from socket import gaierror - - old_url = db.url - with raises(ValueError): - db.set_connection("bolt://user:password@6.6.6.6.6.6.6.6:7687") - APerson(name="New guy 2").save() - db.set_connection(old_url) - # set connection back - assert APerson(name="New guy 3").save() - - @db.transaction.with_bookmark def in_a_tx(*names): for n in names: