From 2cfffbd33d319761ac9a5a67cfe259502e85c179 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 15:17:53 -0800 Subject: [PATCH 1/8] Test against pysqlite3 running SQLite 3.37 Refs #346 and #344. --- .github/workflows/test.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fec8cfa36..025229e39 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,9 @@ jobs: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] numpy: [0, 1] os: [ubuntu-latest, macos-latest, windows-latest] + include: + - os: ubuntu-latest + pysqlite3_sqlite_3_37: true steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -32,6 +35,19 @@ jobs: - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy + - name: Optionall install pysqlite3 with SQLite 3.37 + if: matrix.pysqlite3_sqlite_3_37 + run: |- + cd /tmp + mkdir sqlite-3.37 + cd sqlite-3.37 + wget 'https://www.sqlite.org/2021/sqlite-amalgamation-3370000.zip' + unzip sqlite-amalgamation-3370000.zip + git clone https://github.com/coleifer/pysqlite3/ + cp sqlite-amalgamation-3370000/sqlite3.[ch] pysqlite3 + cd pysqlite3 + python setup.py build_static build bdist_wheel + pip install /tmp/sqlite-3.37/pysqlite3/dist/*.whl - name: Run tests run: | pytest -v From e02c8f35d4ce47c22ac34ffce49f2e6932a65cbf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 15:20:32 -0800 Subject: [PATCH 2/8] Fixes for new matrix option, refs #347 --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 025229e39..bbbdac940 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,9 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] include: - os: ubuntu-latest - pysqlite3_sqlite_3_37: true + python-version: "3.10" + numpy: 0 + pysqlite3_sqlite_3_37: 1 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} @@ -36,7 +38,7 @@ jobs: if: matrix.numpy == 1 run: pip install numpy - name: Optionall install pysqlite3 with SQLite 3.37 - if: matrix.pysqlite3_sqlite_3_37 + if: matrix.pysqlite3_sqlite_3_37 == 1 run: |- cd /tmp mkdir sqlite-3.37 @@ -46,6 +48,7 @@ jobs: git clone https://github.com/coleifer/pysqlite3/ cp sqlite-amalgamation-3370000/sqlite3.[ch] pysqlite3 cd pysqlite3 + pip install wheel python setup.py build_static build bdist_wheel pip install /tmp/sqlite-3.37/pysqlite3/dist/*.whl - name: Run tests From c91f1e6957fc7f3678a04f23d969e587ed4eacb9 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 15:29:23 -0800 Subject: [PATCH 3/8] iterdump() workaround for compatibility with pysqlite3 --- sqlite_utils/cli.py | 7 ++++--- sqlite_utils/utils.py | 1 + tests/test_analyze_tables.py | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index cd0be829b..c032bec36 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -19,6 +19,7 @@ from .utils import ( file_progress, find_spatialite, + iterdump, sqlite3, decode_base64_values, progressbar, @@ -326,7 +327,7 @@ def dump(path, load_extension): """Output a SQL dump of the schema and full contents of the database""" db = sqlite_utils.Database(path) _load_extensions(db, load_extension) - for line in db.conn.iterdump(): + for line in iterdump(db.conn): click.echo(line) @@ -1320,7 +1321,7 @@ def memory( return if dump: - for line in db.conn.iterdump(): + for line in iterdump(db.conn): click.echo(line) return @@ -1330,7 +1331,7 @@ def memory( if save: db2 = sqlite_utils.Database(save) - for line in db.conn.iterdump(): + for line in iterdump(db.conn): db2.execute(line) return diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 00a3c02b5..4116b5f66 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -5,6 +5,7 @@ import io import json import os +from sqlite3.dump import _iterdump as iterdump from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type import click diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 5795a7a1b..2df2698c6 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,5 +1,6 @@ from sqlite_utils.db import Database, ColumnDetails from sqlite_utils import cli +from sqlite_utils.utils import iterdump from click.testing import CliRunner import pytest import sqlite3 @@ -79,7 +80,7 @@ def test_analyze_column(db_to_analyze, column, expected): def db_to_analyze_path(db_to_analyze, tmpdir): path = str(tmpdir / "test.db") db = sqlite3.connect(path) - db.executescript("\n".join(db_to_analyze.conn.iterdump())) + db.executescript("\n".join(iterdump(db_to_analyze.conn))) return path From 93b059dd230eae9eaae472b7fbabd4a66feeb79d Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 15:33:02 -0800 Subject: [PATCH 4/8] Ignore mypy warning about sqlite3.dump --- sqlite_utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index 4116b5f66..d2e14b516 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -5,7 +5,7 @@ import io import json import os -from sqlite3.dump import _iterdump as iterdump +from sqlite3.dump import _iterdump as iterdump # type: ignore from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type import click From f990e134aa8219b687ff6c261330f36824b5df36 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 15:39:53 -0800 Subject: [PATCH 5/8] Need both mypy and flake8 to ignore this line --- sqlite_utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index d2e14b516..e326789cd 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -5,7 +5,7 @@ import io import json import os -from sqlite3.dump import _iterdump as iterdump # type: ignore +from sqlite3.dump import _iterdump as iterdump # type: ignore # noqa: F401 from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type import click From 71b6c3803ecfeb169faee6bb00fdf314a49a97f2 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 15:41:49 -0800 Subject: [PATCH 6/8] And now a fix for Black --- sqlite_utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index e326789cd..681883100 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -5,7 +5,7 @@ import io import json import os -from sqlite3.dump import _iterdump as iterdump # type: ignore # noqa: F401 +from sqlite3.dump import _iterdump as iterdump # type: ignore # noqa: F401 from typing import cast, BinaryIO, Iterable, Optional, Tuple, Type import click From 4beb87f44694a1f82ab873ca1858e9d451afcdcf Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 29 Nov 2021 15:43:04 -0800 Subject: [PATCH 7/8] Fixed typo in workflow --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bbbdac940..b3f9dbe0d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: - name: Optionally install numpy if: matrix.numpy == 1 run: pip install numpy - - name: Optionall install pysqlite3 with SQLite 3.37 + - name: Optionally install pysqlite3 with SQLite 3.37 if: matrix.pysqlite3_sqlite_3_37 == 1 run: |- cd /tmp @@ -45,7 +45,7 @@ jobs: cd sqlite-3.37 wget 'https://www.sqlite.org/2021/sqlite-amalgamation-3370000.zip' unzip sqlite-amalgamation-3370000.zip - git clone https://github.com/coleifer/pysqlite3/ + git clone https://github.com/coleifer/pysqlite3 cp sqlite-amalgamation-3370000/sqlite3.[ch] pysqlite3 cd pysqlite3 pip install wheel From 1a7ef2fe2064ace01d5535fb771f941296fb642a Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 10 Dec 2021 16:59:37 -0800 Subject: [PATCH 8/8] Better test for rebuild, refs #354 --- tests/test_fts.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/tests/test_fts.py b/tests/test_fts.py index 6aa6fbef0..0ae2a52b8 100644 --- a/tests/test_fts.py +++ b/tests/test_fts.py @@ -254,13 +254,12 @@ def test_disable_fts(fresh_db, create_triggers): assert ["searchable"] == fresh_db.table_names() -@pytest.mark.parametrize("table_to_fix", ["searchable", "searchable_fts"]) -def test_rebuild_fts(fresh_db, table_to_fix): +def test_rebuild_fts(fresh_db): table = fresh_db["searchable"] table.insert(search_records[0]) table.enable_fts(["text", "country"]) # Run a search - rows = list(table.search("tanuki")) + rows = list(table.search("are")) assert len(rows) == 1 assert { "rowid": 1, @@ -268,21 +267,14 @@ def test_rebuild_fts(fresh_db, table_to_fix): "country": "Japan", "not_searchable": "foo", }.items() <= rows[0].items() - # Delete from searchable_fts_data - fresh_db["searchable_fts_data"].delete_where() - # This should have broken the index - with pytest.raises(sqlite3.DatabaseError): - list(table.search("tanuki")) + # Insert another record + table.insert(search_records[1]) + # This should NOT show up in a searchs + assert len(list(table.search("are"))) == 1 # Running rebuild_fts() should fix it - fresh_db[table_to_fix].rebuild_fts() - rows2 = list(table.search("tanuki")) - assert len(rows2) == 1 - assert { - "rowid": 1, - "text": "tanuki are running tricksters", - "country": "Japan", - "not_searchable": "foo", - }.items() <= rows2[0].items() + table.rebuild_fts() + rows2 = list(table.search("are")) + assert len(rows2) == 2 @pytest.mark.parametrize("invalid_table", ["does_not_exist", "not_searchable"])