diff --git a/duckdb_engine/__init__.py b/duckdb_engine/__init__.py
index 8cea1a913..4b114afb9 100644
--- a/duckdb_engine/__init__.py
+++ b/duckdb_engine/__init__.py
@@ -24,11 +24,13 @@
 from sqlalchemy.engine.default import DefaultDialect
 from sqlalchemy.engine.url import URL
 from sqlalchemy.ext.compiler import compiles
+from packaging.version import Version
 
 from .config import apply_config, get_core_config
 from .datatypes import ISCHEMA_NAMES, register_extension_types
 
 __version__ = "0.9.2"
+sqlalchemy_version = Version(sqlalchemy.__version__)
 
 if TYPE_CHECKING:
     from sqlalchemy.base import Connection
@@ -39,7 +41,7 @@
 
 
 class DBAPI:
-    paramstyle = duckdb.paramstyle
+    paramstyle = "numeric_dollar" if sqlalchemy_version >= "2.0.0" else "qmark"
     apilevel = duckdb.apilevel
     threadsafety = duckdb.threadsafety
 
diff --git a/duckdb_engine/tests/test_basic.py b/duckdb_engine/tests/test_basic.py
index b474de0f8..d460281c2 100644
--- a/duckdb_engine/tests/test_basic.py
+++ b/duckdb_engine/tests/test_basic.py
@@ -444,6 +444,5 @@ def test_361(engine: Engine) -> None:
             select(date_part)
             .select_from(test)
             .group_by(date_part)
-            .compile(dialect=engine.dialect, compile_kwargs={"literal_binds": True})
         )
         conn.execute(stmt).fetchall()