html_url,issue_url,id,node_id,user,created_at,updated_at,author_association,body,reactions,issue,performed_via_github_app https://github.com/simonw/datasette/issues/1959#issuecomment-1353705072,https://api.github.com/repos/simonw/datasette/issues/1959,1353705072,IC_kwDOBm6k_c5Qr-Zw,9599,2022-12-15T21:04:07Z,2022-12-15T21:04:07Z,OWNER,I'm going to start by getting every test that uses the raw `(app_client)` fixture and nothing else (194 at the moment) to switch to `async def` using a shared Datasette instance and `datasette.client.get()`.,"{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664, https://github.com/simonw/datasette/issues/1959#issuecomment-1353707828,https://api.github.com/repos/simonw/datasette/issues/1959,1353707828,IC_kwDOBm6k_c5Qr_E0,9599,2022-12-15T21:06:29Z,2022-12-15T21:06:29Z,OWNER,"Previous, abandoned attempt at this work (for #1843): ```diff diff --git a/datasette/app.py b/datasette/app.py index 7e682498..cf35c3a2 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -228,7 +228,7 @@ class Datasette: template_dir=None, plugins_dir=None, static_mounts=None, - memory=False, + memory=None, settings=None, secret=None, version_note=None, @@ -238,6 +238,7 @@ class Datasette: nolock=False, ): self._startup_invoked = False + self._extra_on_startup = [] assert config_dir is None or isinstance( config_dir, Path ), ""config_dir= should be a pathlib.Path"" @@ -278,7 +279,7 @@ class Datasette: raise self.crossdb = crossdb self.nolock = nolock - if memory or crossdb or not self.files: + if memory or crossdb or (not self.files and memory is not False): self.add_database( Database(self, is_mutable=False, is_memory=True), name=""_memory"" ) @@ -391,6 +392,9 @@ class Datasette: self._root_token = secrets.token_hex(32) self.client = DatasetteClient(self) + def _add_on_startup(self, fn): + self._extra_on_startup.append(fn) + async def refresh_schemas(self): if self._refresh_schemas_lock.locked(): return @@ -431,6 +435,8 @@ class Datasette: # This must be called for Datasette to be in a usable state if self._startup_invoked: return + for fn in self._extra_on_startup: + await fn() # Register permissions, but watch out for duplicate name/abbr names = {} abbrs = {} @@ -1431,9 +1437,9 @@ class Datasette: ) if self.setting(""trace_debug""): asgi = AsgiTracer(asgi) - asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) for wrapper in pm.hook.asgi_wrapper(datasette=self): asgi = wrapper(asgi) + asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup]) return asgi diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index 56690251..986755cb 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -423,9 +423,9 @@ class AsgiFileDownload: class AsgiRunOnFirstRequest: - def __init__(self, asgi, on_startup): + def __init__(self, app, on_startup): assert isinstance(on_startup, list) - self.asgi = asgi + self.app = app self.on_startup = on_startup self._started = False @@ -434,4 +434,4 @@ class AsgiRunOnFirstRequest: self._started = True for hook in self.on_startup: await hook() - return await self.asgi(scope, receive, send) + return await self.app(scope, receive, send) diff --git a/tests/conftest.py b/tests/conftest.py index cd735e12..d1301943 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,6 +23,15 @@ UNDOCUMENTED_PERMISSIONS = { } +# @pytest.fixture(autouse=True) +# def log_name_of_test_before_test(request): +# # To help identify tests that are hanging +# name = str(request.node) +# with open(""/tmp/test.log"", ""a"") as f: +# f.write(name + ""\n"") +# yield + + def pytest_report_header(config): return ""SQLite: {}"".format( sqlite3.connect("":memory:"").execute(""select sqlite_version()"").fetchone()[0] diff --git a/tests/fixtures.py b/tests/fixtures.py index a6700239..18d3f1b7 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -101,6 +101,19 @@ EXPECTED_PLUGINS = [ ] +def _populate_connection(conn): + # Drop any tables and views that exist + to_drop = conn.execute( + ""SELECT name, type FROM sqlite_master where type in ('table', 'view')"" + ).fetchall() + for name, type in to_drop: + conn.execute(f""DROP {type} IF EXISTS [{name}]"") + conn.executescript(TABLES) + for sql, params in TABLE_PARAMETERIZED_SQL: + with conn: + conn.execute(sql, params) + + @contextlib.contextmanager def make_app_client( sql_time_limit_ms=None, @@ -117,45 +130,22 @@ def make_app_client( metadata=None, crossdb=False, ): - with tempfile.TemporaryDirectory() as tmpdir: - filepath = os.path.join(tmpdir, filename) - if is_immutable: - files = [] - immutables = [filepath] - else: - files = [filepath] - immutables = [] - conn = sqlite3.connect(filepath) - conn.executescript(TABLES) - for sql, params in TABLE_PARAMETERIZED_SQL: - with conn: - conn.execute(sql, params) - # Close the connection to avoid ""too many open files"" errors - conn.close() - if extra_databases is not None: - for extra_filename, extra_sql in extra_databases.items(): - extra_filepath = os.path.join(tmpdir, extra_filename) - c2 = sqlite3.connect(extra_filepath) - c2.executescript(extra_sql) - c2.close() - # Insert at start to help test /-/databases ordering: - files.insert(0, extra_filepath) - os.chdir(os.path.dirname(filepath)) - settings = settings or {} - for key, value in { - ""default_page_size"": 50, - ""max_returned_rows"": max_returned_rows or 100, - ""sql_time_limit_ms"": sql_time_limit_ms or 200, - # Default is 3 but this results in ""too many open files"" - # errors when running the full test suite: - ""num_sql_threads"": 1, - }.items(): - if key not in settings: - settings[key] = value + settings = settings or {} + for key, value in { + ""default_page_size"": 50, + ""max_returned_rows"": max_returned_rows or 100, + ""sql_time_limit_ms"": sql_time_limit_ms or 200, + # Default is 3 but this results in ""too many open files"" + # errors when running the full test suite: + ""num_sql_threads"": 1, + }.items(): + if key not in settings: + settings[key] = value + # We can use an in-memory database, but only if we're not doing anything + # with is_immutable or extra_databases and filename is the default + if not is_immutable and not extra_databases and filename == ""fixtures.db"": ds = Datasette( - files, - immutables=immutables, - memory=memory, + memory=memory or False, cors=cors, metadata=metadata or METADATA, plugins_dir=PLUGINS_DIR, @@ -165,12 +155,57 @@ def make_app_client( template_dir=template_dir, crossdb=crossdb, ) + db = ds.add_memory_database(""fixtures"") + + async def populate_fixtures(): + print(""Here we go... populating fixtures"") + await db.execute_write_fn(_populate_connection) + + ds._add_on_startup(populate_fixtures) yield TestClient(ds) - # Close as many database connections as possible - # to try and avoid too many open files error - for db in ds.databases.values(): - if not db.is_memory: - db.close() + else: + with tempfile.TemporaryDirectory() as tmpdir: + filepath = os.path.join(tmpdir, filename) + if is_immutable: + files = [] + immutables = [filepath] + else: + files = [filepath] + immutables = [] + + conn = sqlite3.connect(filepath) + _populate_connection(conn) + # Close the connection to reduce ""too many open files"" errors + conn.close() + + if extra_databases is not None: + for extra_filename, extra_sql in extra_databases.items(): + extra_filepath = os.path.join(tmpdir, extra_filename) + c2 = sqlite3.connect(extra_filepath) + c2.executescript(extra_sql) + c2.close() + # Insert at start to help test /-/databases ordering: + files.insert(0, extra_filepath) + os.chdir(os.path.dirname(filepath)) + ds = Datasette( + files, + immutables=immutables, + memory=memory, + cors=cors, + metadata=metadata or METADATA, + plugins_dir=PLUGINS_DIR, + settings=settings, + inspect_data=inspect_data, + static_mounts=static_mounts, + template_dir=template_dir, + crossdb=crossdb, + ) + yield TestClient(ds) + # Close as many database connections as possible + # to try and avoid too many open files error + for db in ds.databases.values(): + if not db.is_memory: + db.close() @pytest.fixture(scope=""session"") diff --git a/tests/test_cli.py b/tests/test_cli.py index d3e015fa..d9e4e457 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,6 @@ from .fixtures import ( app_client, + app_client_with_cors, make_app_client, TestClient as _TestClient, EXPECTED_PLUGINS, @@ -38,7 +39,7 @@ def test_inspect_cli(app_client): assert expected_count == database[""tables""][table_name][""count""] -def test_inspect_cli_writes_to_file(app_client): +def test_inspect_cli_writes_to_file(app_client_with_cors): runner = CliRunner() result = runner.invoke( cli, [""inspect"", ""fixtures.db"", ""--inspect-file"", ""foo.json""] ```","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664, https://github.com/simonw/datasette/issues/1959#issuecomment-1353720559,https://api.github.com/repos/simonw/datasette/issues/1959,1353720559,IC_kwDOBm6k_c5QsCLv,9599,2022-12-15T21:19:56Z,2022-12-15T21:19:56Z,OWNER,"Here's a port of the first `def ...(app_client)` test. Note that the TestClient object works slightly differently from the HTTPX response returned by `await datasette.client.get(...)`: ```diff diff --git a/datasette/app.py b/datasette/app.py index f3cb8876..b770b469 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -281,7 +281,7 @@ class Datasette: raise self.crossdb = crossdb self.nolock = nolock - if memory or crossdb or not self.files: + if memory or crossdb or (not self.files and memory is not False): self.add_database( Database(self, is_mutable=False, is_memory=True), name=""_memory"" ) diff --git a/pytest.ini b/pytest.ini index 559e518c..0bcb0d1e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,4 +8,5 @@ filterwarnings= ignore:.*current_task.*:PendingDeprecationWarning markers = serial: tests to avoid using with pytest-xdist + ds_client: tests using the ds_client fixture asyncio_mode = strict diff --git a/tests/conftest.py b/tests/conftest.py index cd735e12..648423ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import httpx import os import pathlib import pytest +import pytest_asyncio import re import subprocess import tempfile @@ -23,6 +24,22 @@ UNDOCUMENTED_PERMISSIONS = { } +@pytest_asyncio.fixture +async def ds_client(): + from datasette.app import Datasette + from .fixtures import METADATA, PLUGINS_DIR + ds = Datasette(memory=False, metadata=METADATA, plugins_dir=PLUGINS_DIR) + from .fixtures import TABLES, TABLE_PARAMETERIZED_SQL + db = ds.add_memory_database(""fixtures"") + def prepare(conn): + conn.executescript(TABLES) + for sql, params in TABLE_PARAMETERIZED_SQL: + with conn: + conn.execute(sql, params) + await db.execute_write_fn(prepare) + return ds.client + + def pytest_report_header(config): return ""SQLite: {}"".format( sqlite3.connect("":memory:"").execute(""select sqlite_version()"").fetchone()[0] diff --git a/tests/test_api.py b/tests/test_api.py index 5f2a6ea6..ddf4219c 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -23,12 +23,15 @@ import sys import urllib -def test_homepage(app_client): - response = app_client.get(""/.json"") - assert response.status == 200 +@pytest.mark.ds_client +@pytest.mark.asyncio +async def test_homepage(ds_client): + response = await ds_client.get(""/.json"") + assert response.status_code == 200 assert ""application/json; charset=utf-8"" == response.headers[""content-type""] - assert response.json.keys() == {""fixtures"": 0}.keys() - d = response.json[""fixtures""] + data = response.json() + assert data.keys() == {""fixtures"": 0}.keys() + d = data[""fixtures""] assert d[""name""] == ""fixtures"" assert d[""tables_count""] == 24 assert len(d[""tables_and_views_truncated""]) == 5 @@ -36,7 +39,7 @@ def test_homepage(app_client): # 4 hidden FTS tables + no_primary_key (hidden in metadata) assert d[""hidden_tables_count""] == 6 # 201 in no_primary_key, plus 6 in other hidden tables: - assert d[""hidden_table_rows_sum""] == 207, response.json + assert d[""hidden_table_rows_sum""] == 207, data assert d[""views_count""] == 4 ```","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664, https://github.com/simonw/datasette/issues/1959#issuecomment-1353721091,https://api.github.com/repos/simonw/datasette/issues/1959,1353721091,IC_kwDOBm6k_c5QsCUD,9599,2022-12-15T21:20:32Z,2022-12-15T21:20:32Z,OWNER,Rather than tediously rewriting every single test to the new shape I'm going to try a wrapper for that HTTPX response that transforms it into an imitation of the one returned by the existing `TestClient` class.,"{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664, https://github.com/simonw/datasette/issues/1959#issuecomment-1353728682,https://api.github.com/repos/simonw/datasette/issues/1959,1353728682,IC_kwDOBm6k_c5QsEKq,9599,2022-12-15T21:28:35Z,2022-12-15T21:28:35Z,OWNER,"Got this error trying to have two tests use the same `ds_client` async fixture when I added `scope=""session""` to that fixture: - https://github.com/tortoise/tortoise-orm/issues/638 Adding this to `conftest.py` (as suggested in that issue thread) seemed to fix it: ```python @pytest.fixture(scope=""session"") def event_loop(): return asyncio.get_event_loop() ```","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664, https://github.com/simonw/datasette/issues/1959#issuecomment-1353738075,https://api.github.com/repos/simonw/datasette/issues/1959,1353738075,IC_kwDOBm6k_c5QsGdb,9599,2022-12-15T21:35:56Z,2022-12-15T21:35:56Z,OWNER,"I built that `OldResponse` class: ```diff diff --git a/tests/utils.py b/tests/utils.py index 191ead9b..f39ac434 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -30,3 +30,25 @@ def inner_html(soup): def has_load_extension(): conn = sqlite3.connect("":memory:"") return hasattr(conn, ""enable_load_extension"") + + +class OldResponse: + ""Transform an HTTPX response to simulate the older TestClient responses"" + # https://github.com/simonw/datasette/issues/1959#issuecomment-1353721091 + def __init__(self, response): + self.response = response + self._json = None + + @property + def headers(self): + return self.response.headers + + @property + def status(self): + return self.response.status_code + + @property + def json(self): + if self._json is None: + self._json = self.response.json() + return self._json ``` I can use it in tests like this: ```python @pytest.mark.asyncio async def test_homepage(ds_client): response = OldResponse(await ds_client.get(""/.json"")) assert response.status == 200 assert ""application/json; charset=utf-8"" == response.headers[""content-type""] assert response.json.keys() == {""fixtures"": 0}.keys() d = response.json[""fixtures""] assert d[""name""] == ""fixtures"" assert d[""tables_count""] == 24 assert len(d[""tables_and_views_truncated""]) == 5 assert d[""tables_and_views_more""] is True # 4 hidden FTS tables + no_primary_key (hidden in metadata) assert d[""hidden_tables_count""] == 6 # 201 in no_primary_key, plus 6 in other hidden tables: assert d[""hidden_table_rows_sum""] == 207, response.json assert d[""views_count""] == 4 ``` But as I work through the tests I'm finding it's actually not too hard to port them over, so I likely won't use it after all.","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664, https://github.com/simonw/datasette/issues/1959#issuecomment-1353747370,https://api.github.com/repos/simonw/datasette/issues/1959,1353747370,IC_kwDOBm6k_c5QsIuq,9599,2022-12-15T21:45:14Z,2022-12-15T21:45:14Z,OWNER,I'm going to do this in a PR.,"{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664, https://github.com/simonw/datasette/issues/1959#issuecomment-1355317782,https://api.github.com/repos/simonw/datasette/issues/1959,1355317782,IC_kwDOBm6k_c5QyIIW,9599,2022-12-16T17:57:25Z,2022-12-16T17:57:25Z,OWNER,"Opened a follow-up issue here: - #1962","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664, https://github.com/simonw/datasette/issues/1959#issuecomment-1356478792,https://api.github.com/repos/simonw/datasette/issues/1959,1356478792,IC_kwDOBm6k_c5Q2jlI,9599,2022-12-17T21:49:36Z,2022-12-17T21:49:36Z,OWNER,"Made a really good start on this in the just-merged PR: - #1960 The follow-up work will happen in: - #1962","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1499081664,