Also it should link to foreign keys like the table page does.

I'm going to wrap this up in a plugin for the moment - I want it in Datasette core but I'd like to improve the implementation first with things like support for base_url which will likely depend on #1533 or similar.

@hannseman
I have tried the above suggestion given by you but somehow I'm getting the below error.

note : I'm running my application with Docker.

app_1 | {"event": "Exception in ASGI application\n", "exc_info": ["<class 'RuntimeError'>", "RuntimeError('no running event loop')", "<traceback object at 0x7f9d2f5dffc0>"], "logger": "uvicorn.error", "level": "error", "timestamp": "2021-12-02T12:06:36.011448Z"}

I realized you couldn't click the links any more because the hovercard overlapped them, so I changed it to this instead. Need to reconsider the when-to-hide logic though.

hovercard.style.top = (ev.pageY + 15) + 'px';

Needs pageX not clientX because otherwise it doesn't work when you scroll down the page.

This is a bit of a mess but it does keep the hovercard around for a moment and then fade it away when you mouse out of it:

{% extends "base.html" %}

{% block content %}

<h1>Hovercards demo</h1>

Here is a <a href="/fixtures/facetable/1">link to a row</a>

<script>
let hovercardOuterAnimation = null;
let hovercardInnerAnimation = null;
let hovercard = document.createElement("div");
hovercard.setAttribute("id", "datasette-hovercard")
hovercard.style.width = '300px';
hovercard.style.height = '200px';
hovercard.style.overflow = 'auto';
hovercard.style.backgroundColor = 'white';
hovercard.style.border = '1px solid #ccc';
hovercard.style.padding = '10px';
hovercard.style.position = 'absolute';
hovercard.style.whiteSpace = 'pre';
hovercard.style.display = 'none';
hovercard.style.boxShadow = '1px 2px 8px 2px rgba(0,0,0,0.08)';
document.body.appendChild(hovercard);

document.addEventListener("mouseover", async (ev) => {
  const a = ev.target;
  if (a.nodeName != 'A') {
    return;
  }
  // TODO: Respect base_url and suchlike
  if (a.pathname.split("/").length != 4) {
    return; // Definitely not a row
  }
  // OK, it might be a row! Try a fetch
  let row;
  if (a.hovercardRowData) {
    row = a.hovercardRowData;
  } else {
    const response = await fetch(a.pathname + ".json?_shape=array");
    if (response.status == 200) {
      const data = await response.json();
      row = data[0];
      a.hovercardRowData = row;
    }
  }
  if (row) {
    // Cancel any existing animations
    if (hovercardOuterAnimation) {
      clearTimeout(hovercardOuterAnimation);
    }
    if (hovercardInnerAnimation) {
      clearTimeout(hovercardInnerAnimation);
    }
    // -15 so mouse pointer starts in the box
    hovercard.style.top = (ev.pageY - 15) + 'px';
    hovercard.style.left = (ev.pageX - 15) + 'px';
    hovercard.innerText = JSON.stringify(row, null, 4);
    hovercard.style.display = 'block';
    hovercard.style.opacity = 100;
    hovercard.style.transition = 'none';
  }
});

document.addEventListener("mouseout", (ev) => {
  if (ev.target.id != "datasette-hovercard") {
    return;
  }
  hovercardOuterAnimation = setTimeout(() => {
    hovercard.style.transition = 'opacity 0.4s ease-in-out';
    hovercard.style.opacity = 0;
    hovercardInnerAnimation = setTimeout(() => {
      hovercard.style.transition = 'none';
      hovercard.style.display = "none";
      hovercard.style.opacity = 100;
    }, 800)
  }, 400);
})
</script>

{% endblock %}

A first prototype (saved as templates/pages/hovercard.html and run with datasette fixtures.db --template-dir=templates):

{% extends "base.html" %}

{% block content %}

<h1>Hovercards demo</h1>

Here is a <a href="/fixtures/facetable/1">link to a row</a>

<script>
document.addEventListener("mouseover", async (ev) => {
  const a = ev.target;
  if (a.nodeName != 'A') {
    return;
  }
  window.A = a;
  // TODO: Respect base_url and suchlike
  if (A.pathname.split("/").length != 4) {
    return; // Definitely not a row
  }
  // OK, it might be a row! Try a fetch
  const response = await fetch(A.pathname + ".json?_shape=array");
  if (response.status == 200) {
    const data = await response.json();
    const row = data[0];
    if (!a.hovercard) {
      a.hovercard = document.createElement("div");
      a.hovercard.style.width = '300px';
      a.hovercard.style.height = '200px';
      a.hovercard.style.overflow = 'auto';
      a.hovercard.style.backgroundColor = 'white';
      a.hovercard.style.border = '1px solid #ccc';
      a.hovercard.style.padding = '10px';
      a.hovercard.style.position = 'absolute';
      a.hovercard.style.whiteSpace = 'pre';
      a.hovercard.style.top = ev.clientY + 'px';
      a.hovercard.style.left = ev.clientX + 'px';
      a.hovercard.style.boxShadow = '1px 2px 8px 2px rgba(0,0,0,0.08)';
      document.body.appendChild(a.hovercard);
    }
    a.hovercard.innerText = JSON.stringify(row, null, 4);
    a.hovercard.style.display = 'block';
  }
});
</script>

{% endblock %}

Lots of decisions to make here. Most importantly, when should it be hidden again?

The layout of the hover card could be similar to the one used by datasette-cluster-map:

https://user-images.githubusercontent.com/9599/144300723-2c733a0f-9daf-430d-a247-6c87f138e728.png">

thanks so very much for the prompt attention and fix! Plus, the animated GIF showing the bug is just extra and I love it. Interactions like this are why I love open source.

I'd use this. Right now, I tend to do touch my.db and then enable-wal or whatever else, but I'm never sure if that's a bad idea.

Potential other options could include:

• --page-size https://www.sqlite.org/pragma.html#pragma_page_size

Maybe this:

sqlite-utils create-database my.db

With options that include --enable-wal (actually that's the only option I can think of).

This is consistent with the existing create-table and create-view commands.

It's a really great API and the documentation is really great too. Honestly, in more than 20 years of professional experience, I haven't worked with any software API that was more of a joy to use.

These two pages now help demonstrate the fix:

• https://latest.datasette.io/fixtures/facet_cities/1
• https://latest.datasette.io/fixtures/attraction_characteristic/2

I added a new test for these here:

https://github.com/simonw/datasette/blob/35b12746ba2bf9f254791bddac03d25b19be9b77/tests/test_html.py#L823-L848

That's really cool to hear - I've not seen many people actively building on top of the JSON API.

Manually tested this too, looks like that fixed it.

I started fiddling with a test for this which extracts the <select> options and ALL of the <input> fields, but I probably won't use it:

def test_exact_parameter_results_in_correct_hidden_fields(app_client):
    # https://github.com/simonw/datasette/issues/1527
    response = app_client.get(
        "/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown"
    )
    # In this case we should NOT have a hidden _neighborhood__exact=Downtown field
    form = Soup(response.body, "html.parser").find("form")
    selects = [
        {
            "name": select["name"],
            "value": select.select("option[selected]")[0].text
            if select.select("option[selected]")
            else "",
        }
        for select in form.findAll("select")
    ]
    inputs = [input.attrs for input in form.findAll("input")]
    # Turn those both into a {name: (value, type)} array
    form_inputs = {}
    form_inputs.update(
        {select["name"]: (select["value"], "select") for select in selects}
    )
    form_inputs.update(
        {
            input["name"]: (input.get("value"), input["type"])
            for input in inputs
            if input.get("name")
        }
    )
    assert form_inputs == {
        "_filter_column_1": ("_neighborhood", "select"),
        "_filter_op_1": ("=", "select"),
        "_filter_value_1": ("Downtown", "text"),
        "_filter_column": ("", "select"),
        "_filter_op": ("", "select"),
        "_filter_value": (None, "text"),
        "_sort": ("Sort by pk", "select"),
        "_sort_by_desc": (None, "checkbox"),
        "_facet": ("_neighborhood", "hidden"),
        "_neighborhood__exact": ("Downtown", "hidden"),
    }

The problem is that last hidden field, _neighborhood__exact=Downtown - which should not be there.

Codecov Report

Merging #1529 (2cb3dbb) into main (48f1199) will decrease coverage by 0.06%.
The diff coverage is n/a.

:exclamation: Current head 2cb3dbb differs from pull request most recent head 495b726. Consider uploading reports for the commit 495b726 to get more accurate results

@@            Coverage Diff             @@
##             main    #1529      +/-   ##
==========================================
- Coverage   91.90%   91.83%   -0.07%     
==========================================
  Files          34       34              
  Lines        4434     4434              
==========================================
- Hits         4075     4072       -3     
- Misses        359      362       +3     

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 1beb7d9...495b726. Read the comment docs.

If I'm going to skipIf() those tests I need a way to check if pysqlite3 is being used.

A short term fix would be to skip those tests against pysqlite3 - but longer term it would be good to address the underlying issue, particularly for the WAL ones (the FTS ones aren't too worrying since if you deliberately try and break the FTS table it's not hugely problematic if you corrupt your database).

Some interesting test failures in the version that runs with pysqlite3:

=========================== short test summary info ============================
FAILED tests/test_cli.py::test_enable_wal - assert 0 == 1
FAILED tests/test_cli.py::test_disable_wal - pysqlite3.dbapi2.OperationalErro...
FAILED tests/test_fts.py::test_rebuild_fts[searchable] - pysqlite3.dbapi2.Dat...
FAILED tests/test_fts.py::test_rebuild_fts[searchable_fts] - pysqlite3.dbapi2...
FAILED tests/test_wal.py::test_enable_disable_wal - pysqlite3.dbapi2.Operatio...
================== 5 failed, 750 passed, 3 skipped in 15.20s ===================

https://github.com/simonw/sqlite-utils/runs/4360759085

The WAL errors look like this:

E           pysqlite3.dbapi2.OperationalError: cannot change into wal mode from within a transaction

Triggered by a call to db.enable_wal()

The FTS errors are caused by tests that try to deliberately corrupt the FTS index by running fresh_db["searchable_fts_data"].delete_where() - and then rebuilding it using rebuild_fts():

    @pytest.mark.parametrize("table_to_fix", ["searchable", "searchable_fts"])
    def test_rebuild_fts(fresh_db, table_to_fix):
        table = fresh_db["searchable"]
        table.insert(search_records[0])
        table.enable_fts(["text", "country"])
        # Run a search
        rows = list(table.search("tanuki"))
        assert len(rows) == 1
        assert {
            "rowid": 1,
            "text": "tanuki are running tricksters",
            "country": "Japan",
            "not_searchable": "foo",
        }.items() <= rows[0].items()
        # Delete from searchable_fts_data
        fresh_db["searchable_fts_data"].delete_where()
        # This should have broken the index
        with pytest.raises(sqlite3.DatabaseError):
            list(table.search("tanuki"))
        # Running rebuild_fts() should fix it
>       fresh_db[table_to_fix].rebuild_fts()

tests/test_fts.py:277: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
sqlite_utils/db.py:1947: in rebuild_fts
    self.db.execute(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <Database <pysqlite3.dbapi2.Connection object at 0x7fd95d299c30>>
sql = "INSERT INTO [searchable_fts]([searchable_fts]) VALUES('rebuild');"
parameters = None

    def execute(
        self, sql: str, parameters: Optional[Union[Iterable, dict]] = None
    ) -> sqlite3.Cursor:
        "Execute SQL query and return a ``sqlite3.Cursor``."
        if self._tracer:
            self._tracer(sql, parameters)
        if parameters is not None:
            return self.conn.execute(sql, parameters)
        else:
>           return self.conn.execute(sql)
E           pysqlite3.dbapi2.DatabaseError: database disk image is malformed

Codecov Report

Merging #347 (71b6c38) into main (213a0ff) will increase coverage by 0.00%.
The diff coverage is 100.00%.

@@           Coverage Diff           @@
##             main     #347   +/-   ##
=======================================
  Coverage   96.51%   96.52%           
=======================================
  Files           5        5           
  Lines        2270     2271    +1     
=======================================
+ Hits         2191     2192    +1     
  Misses         79       79           

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 213a0ff...71b6c38. Read the comment docs.

Took a bit of experimenting to get both mypy AND flake8 to ignore the same line. The incantation that worked was this one:

https://github.com/simonw/sqlite-utils/blob/f990e134aa8219b687ff6c261330f36824b5df36/sqlite_utils/utils.py#L8

Order here matters - this did NOT work for both tools:

 from sqlite3.dump import _iterdump as iterdump   # noqa: F401 # type: ignore

https://github.com/simonw/sqlite-utils/blob/93b059dd230eae9eaae472b7fbabd4a66feeb79d/.github/workflows/test.yml#L11-L20

This configuration means that the numpy=0, Python=3.10, os=Ubuntu build will additionally use pysqlite3 with the SQLite 3.37.0.

It's failing right now: https://github.com/simonw/sqlite-utils/runs/4360593156 - because pysqlite3 doesn't provide .iterdump().

I can use the workaround from this comment: https://github.com/coleifer/pysqlite3/issues/24#issuecomment-982081267

It failed on other Python versions with mypy:

sqlite_utils/utils.py:8: error: Cannot find implementation or library stub for module named "sqlite3.dump"
sqlite_utils/utils.py:8: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports

Here's the test run that's installing pysqlite3 and that version of SQLite: https://github.com/simonw/sqlite-utils/runs/4360663292

To keep things simple for the moment I'm only going to add one extra thing to the matrix, and it will be a run of the tests against SQLite 3.37.0 using pysqlite3 on Linux only.

I can use this mechanism: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions#example-including-new-combinations

I have a working recipe for compiling it for macOS here: https://github.com/simonw/sqlite-utils/issues/344#issuecomment-982006544

For the moment I think I'll combine two problems into one, and just add a single matrix alternative that uses pysqlite3 running SQLite 3.37.0 - only on macOS and Linux so I don't have to figure out how to compile it for Windows.

This is the implementation: https://github.com/simonw/sqlite-utils/blob/213a0ff177f23a35f3b235386366ff132eb879f1/sqlite_utils/db.py#L1236-L1241

Documented here: https://sqlite-utils.datasette.io/en/latest/python-api.html#strict

Here's a modified version of the dump command that works even with pysqlite3:

@cli.command()
@click.argument(
    "path",
    type=click.Path(exists=True, file_okay=True, dir_okay=False, allow_dash=False),
    required=True,
)
@load_extension_option
def dump(path, load_extension):
    """Output a SQL dump of the schema and full contents of the database"""
    db = sqlite_utils.Database(path)
    _load_extensions(db, load_extension)
    # pysqlite3 doesn't implement .iterdump()
    from sqlite3.dump import _iterdump

    for line in _iterdump(db.conn):
        click.echo(line)

Ideally I'd like an extra set of matrix options for different versions of SQLite.

I can use pysqlite3 for this, but it isn't a completely compatible drop-in replacement - turns out it doesn't support the iterdump() method for example, see https://github.com/coleifer/pysqlite3/issues/24

Need to figure out a good pattern for testing this in CI too - it will currently skip the new tests if it doesn't have SQLite 3.37 or higher.

I haven't documented db.supports_strict yet (I documented table.strict) because there wasn't an obvious section of the documentation for it.

I need to remember to document it once I add documentation for the strict=True parameter.

I'm going to add that as db.supports_strict.

Here's a function that detects if strict is supported or not:

import secrets
import sqlite3

def supports_strict_tables(db = None):
    if db is None:
        db = sqlite3.connect(":memory:")
    try:
        table_name = 't{}'.format(secrets.token_hex(16))
        with db:
            db.execute("create table {} (name text) strict".format(table_name))
            db.execute("drop table {}".format(table_name))
        return True
    except:
        return False

Made myself a test strict table like so:

>>> import pysqlite3
>>> conn = pysqlite3.connect("/tmp/strict-table.db")
>>> conn.execute("create table foo (id integer primary key, name text, weight real) strict")
<pysqlite3.dbapi2.Cursor object at 0x104317340>
>>> cursor = conn.cursor()
>>> with conn:
...     cursor.execute("insert into foo (name, weight) values ('Lila', '2.31')")
<pysqlite3.dbapi2.Cursor object at 0x10331e1f0>
>>> conn.close()

Uploaded that to: https://static.simonwillison.net/static/2021/strict-table.db

From the STRICT docs:

The SQLite parser accepts a comma-separated list of table options after the final close parenthesis in a CREATE TABLE statement. As of this writing (2021-08-23) only two options are recognized:

• STRICT
• WITHOUT ROWID

So I think I need to read the CREATE TABLE statement from the sqlite_master table, split on the last ), split those tokens on , and see if create is in there (case insensitive).

Is there a need for an introspection function for telling if a SQLite table is in strict mode or not? How would that work?

I'm leaning towards silently ignore if SQLite version doesn't support it. That means that the first time you attempt to use strict=True we will need to run a test against the database connection to see what version of SQLite it uses, then cache the result to avoid making the same call again for the same connection.

What should happen if you attempt to use strict=True against a SQLite version prior to 3.37.0?

An obvious error would be best... but how about silently ignoring it on older versions instead? That would match how we handle deterministic=True for registering functions:

https://github.com/simonw/sqlite-utils/blob/126703706ea153f63e6134ad14e5712e4bbcb8ae/sqlite_utils/db.py#L372-L380

https://github.com/simonw/sqlite-utils/blob/93c7fd9868fed3193a1732b39bfac539e5812b0b/tests/test_register_function.py#L34-L37

There are a few places that the strict=True option could go:

• table.create() and table.create_table_sql()
• The Database() constructor, to turn it on for all created tables
• The .insert() / .insert_all() etc family of methods that can implicitly create tables

I'll definitely implement the first one, and likely the second one too. I'm on the fence with regards to the third one.

This worked:

cd /tmp
mkdir sqlite-3.37
cd sqlite-3.37
wget 'https://www.sqlite.org/2021/sqlite-amalgamation-3370000.zip'
unzip sqlite-amalgamation-3370000.zip
git clone https://github.com/coleifer/pysqlite3/
cp sqlite-amalgamation-3370000/sqlite3.[ch] pysqlite3
cd pysqlite3
python3 setup.py build_static build bdist_wheel

This gave me a file here:

pysqlite3 % ls -l dist  
total 1872
-rw-r--r--  1 simon  wheel  956557 Nov 29 12:38 pysqlite3-0.4.6-cp39-cp39-macosx_10_15_x86_64.whl

That wheel only works when installed for Python 3.9 (it failed to install in a Python 3.10 virtual environment) - but pip install /tmp/sqlite-3.37/pysqlite3/dist/pysqlite3-0.4.6-cp39-cp39-macosx_10_15_x86_64.whl gave me a working pysqlite3 - and the following worked:

>>> import pysqlite3
>>> pysqlite3.connect(":memory:").execute("select sqlite_version()").fetchall()
[('3.37.0',)]

And if I install sqlite-utils in the same virtual environment this works:

% sqlite-utils memory 'select sqlite_version()'
[{"sqlite_version()": "3.37.0"}]

I'm going to build my own pysqlite3 wheel with the latest SQLite to try this out, following the instructions on https://github.com/coleifer/pysqlite3

From that page:

If you try to open a database containing the STRICT keyword in an earlier version of SQLite, it will not recognize the keyword and will report an error (except as noted below.

[...]

Because of a quirk in the SQL language parser, versions of SQLite prior to 3.37.0 can still read and write STRICT tables if they set "PRAGMA writable_schema=ON" immediately after opening the database file, prior to doing anything else that requires knowing the schema.

There isn't any way to do this with sqlite as far as I know. The only option is to insert the right number of ? placeholders into the sql template and then provide an array of values.

FWIW I've written some web components that consume the json api and I think it's a really nice way to work with datasette. I like the combination with datasette+sqlite as a back-end feeding data to a front-end that's entirely javascript + html.

TIL: https://til.simonwillison.net/datasette/reuse-click-for-register-commands

Thanks to https://stackoverflow.com/a/45514541/6083 I found the right pattern:

from datasette import hookimpl
from git_history.cli import cli as git_history_cli

@hookimpl
def register_commands(cli):
    cli.add_command(git_history_cli, name="git-history")

I think this is a little bit too obscure to add to the Datasette documentation - I'll turn it into a TIL instead.

Just trying to use git_history_file produces this error:

TypeError: Attempted to convert a callback into a command twice.

Codecov Report

Merging #1537 (fcfaec1) into main (48f1199) will decrease coverage by 0.06%.
The diff coverage is n/a.

@@            Coverage Diff             @@
##             main    #1537      +/-   ##
==========================================
- Coverage   91.90%   91.83%   -0.07%     
==========================================
  Files          34       34              
  Lines        4434     4434              
==========================================
- Hits         4075     4072       -3     
- Misses        359      362       +3     

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update 48f1199...fcfaec1. Read the comment docs.

(I could experiment with merging the two tables by adding a temporary undocumented ?_sql= parameter to the in-progress table view that sets an alternative query instead of select cols from table - added bonus, this will force me to use introspection against the returned columns rather than mixing in the known columns for the specified table)

Aside: is there any reason this work can't complete the long-running goal of merging the TableView and QueryView, such that most of the features available for tables become available for arbitrary queries too?

I had already mentally committed to implementing facets for queries, but I just realized that filters could work too - using either a CTE or a nested query.

Pagination is the one holdout here, since table pagination uses keyset pagination over a known order. But maybe arbitrary queries can only be paginated off you order them first?

I'm also going to use the new datasette-table Web Component to help guide the design of the new API, which relates directly to this issue too:

• 1532

To new requirements inspired by work on the datasette-table (and datasette-notebook) projects:

• 1533

• 1534

If I'm going to do this, is there value in also spotting Accept: text/csv and returning CSV for that? I'm pretty sure no client has EVER implemented this though, so it feels like it would be showboating.

I built an initial prototype of this in a branch: https://github.com/simonw/datasette/commit/e0a84691c2959f2d1d76948574c9c4a910c7556c - which exposed even more flaws in the way TableView is structured (adding custom HTTP headers to the response is way harder than it should be) which I should address in the refactor in #617.

I just remembered that there's one other place that this could fit: as a Datasette "insert" plugin.

This is vaporware at the moment, but the idea is that Datasette itself could grow a mechanism for importing data, that's driven by plugins.

Out of the box Datasette would be able to import CSV and CSV files, similar to sqlite-utils insert ... --csv - but plugins would then be able to add support for additional format such as GeoJSON or - in this case - Parquet.

The neat thing about having it as a Datasette plugin is that one plugin would enable three different ways of importing data:

1. Via a new datasette insert ... CLI option (similar to sqlite-utils)
2. Via a web form upload interface, where authenticated Datasette users would be able to upload files
3. Via an API interface, where files could be programatically submitted to a running Datasette server

I started fleshing out this idea quite a while ago but didn't make much concrete progress, maybe I should revisit it:

• https://github.com/simonw/datasette/issues/1160

Thanks for your reply @simonw . Tbh, my first attempt was actually the parquet-to-sqlite package but I already had Makefiles that relied on SQLite-utils and it was less intrusive to my workflow. Maybe I'll revisit that decision.
FYI: there's a [sqlite-parquet-vtable](https://github.com/cldellow/sqlite-parquet-vtable)

I don't think plugins make much sense either. Probably defeats the purpose of simplicity: simple database along with a pip-able package.

I tried deploying the most recent version of the Dockerfile in this thread (link to comment), and after trying a few different different combinations, I was only successful when I used --no-cpu-throttling ("CPU Is always allocated" in the UI)

Using this method, I got a very similar issue to you: The first time I'd load the site I'd get a 503. But after that first load, I didn't get the issue again. It would re-occur if the service started from cold boot.

I suspect this is a race condition in the supervisord configuration. The errors I got were the same Connection refused: AH00957: http: attempt to connect to 127.0.0.1:8001 (127.0.0.1) failed, and that seems to indicate that datasette hadn't yet started.

Looking at the order of logs getting back, the processes reported successfully completing loading after the first 503 was returned, so that makes me think race condition.

I can replicate this locally, if I docker run and request localhost:5000/prefix before I get the datasette entered RUNNING state message. Cloud Run wakes up when requests are received, so this test would semi-replicate that, but local docker would be the equivalent of a persistent process, hence it doesn't normally exhibit the same issues.

Unfortunately supervisor/supervisor issue 122 (not linking as to prevent cross-project link spam) seems to say that dependency chaining is a feature that's been asked for for a long time, but hasn't been implemented. You could try some suggestions in that thread.

If you suspect that Cloud Run throttled CPU could be the cause, you can request to have CPU always allocated with gcloud beta run deploy --no-cpu-throttling (read more)

It could also be the Cloud Run sandbox that somehow gets in the way here, in which case I recommend testing with the second generation execution environment: gcloud beta run deploy --execution-environment gen2

Oh, another thing to consider: I believe this would be the first "_file" key in datasette's metadata, compared to other "_url" keys like "license_url" or "about_url". Not too sure what considerations to include with this (ex should missing files cause Datasette to stop before starting, should build scripts bundle these sql files somewhere during datasette package, etc.)

Fixed in the 0.12 release of that plugin: https://github.com/simonw/datasette-publish-vercel/releases/tag/0.12

Addressing that over here (hadn't seen that issue yet, thanks for the prod): https://github.com/simonw/datasette-publish-vercel/issues/51#issuecomment-975073026

I think the root cause is this hidden form field on https://latest.datasette.io/fixtures/facetable?_facet=_neighborhood&_neighborhood__exact=Downtown

            <input type="hidden" name="_facet" value="_neighborhood">
            <input type="hidden" name="_neighborhood__exact" value="Downtown">
        <input type="submit" value="Apply">

https://latest.datasette.io/fixtures/facetable can't quite demonstrate the bug because _neighborhood isn't a foreign key - I should rename city_id to _city_id.

Here's the template: https://github.com/simonw/datasette/blob/48f11998b73350057b74fe6ab464d4ac3071637c/datasette/templates/row.html#L39-L41

Closing this in favour of Datasette Desktop: https://datasette.io/desktop

Some other recent projects (like trying to get this library to work in JupyterLite) have made me much more cautious about adding new dependencies, especially dependencies like pyarrow which require custom C/Rust extensions.

There are a few ways this could work though:

• Have this as an optional dependency feature - so it only works if the user installs pyarrow as well
• Implement this as a separate tool, parquet-to-sqlite - which could itself depend on sqlite-utils
• Add a concept of "plugins" to sqlite-utils, similar to how those work in Datasette: https://docs.datasette.io/en/stable/plugins.html

My favourite option is parquet-to-sqlite because that can be built without any additional changes to sqlite-utils at all!

I find the concept of plugins for sqlite-utils interesting. I've so far not had quite enough potential use-cases to convince me this is worthwhile (especially since it should be very easy to build out separate tools entirely), but I'm ready to be convinced that a plugin mechanism would be worthwhile.

I noticed that http://datasette-apache-proxy-demo.datasette.io/ wasn't redirecting to https so I built a new plugin: https://github.com/simonw/datasette-redirect-to-https

% curl -i 'http://datasette-apache-proxy-demo.datasette.io/prefix/fixtures/no_primary_key' 
HTTP/1.1 301 Moved Permanently
date: Sat, 20 Nov 2021 23:22:50 GMT
server: Fly/51d150d (2021-11-19)
location: https://datasette-apache-proxy-demo.datasette.io/fixtures/no_primary_key
x-proxied-by: Apache2 Debian
transfer-encoding: chunked
via: 1.1 fly.io
fly-request-id: 01FMZTHTHVPC8BZY0625D7JV4B

New TIL: https://til.simonwillison.net/fly/custom-subdomain-fly

I think another thing would be to make /pages/robots.txt work. That way you can use jinja to generate a desired robots.txt. I'm using it to allow the main index and what it links to to be crawled (but not the database pages directly.)

I pointed CNAME of datasette-apache-proxy-demo.datasette.io at datasette-apache-proxy-demo.fly.dev. using Vercel DNS:

https://user-images.githubusercontent.com/9599/142740008-942f180b-bedb-4a44-b6ef-1b0e7fd32416.png">

Then I asked Fly to issue a LetsEncrypt certificate for that:

% flyctl certs create datasette-apache-proxy-demo.datasette.io
# About 53 seconds later:
% flyctl certs show datasette-apache-proxy-demo.datasette.io
The certificate for datasette-apache-proxy-demo.datasette.io has been issued.

Hostname                  = datasette-apache-proxy-demo.datasette.io

DNS Provider              = constellix

Certificate Authority     = Let's Encrypt

Issued                    = ecdsa,rsa

Added to App              = 53 seconds ago

Source                    = fly

https://datasette-apache-proxy-demo.datasette.io/ works now - I'll use that in the documentation.

I'm also going to extract the Apache config files from https://github.com/simonw/datasette/blob/250db8192cb8aba5eb8cd301ccc2a49525bc3d24/demos/apache-proxy/Dockerfile into a separate file to make it easier to read.

(The supervisor config needs to be dynamically constructed to include $DATASETTE_REF so I will leave it where it is.)

I think what's happening here is Apache is actually making a request to /fixtures rather than making a request to /prefix/fixtures - and Datasette is replying to requests on both the prefixed and the non-prefixed paths.

This is pretty confusing! I think Datasette should ONLY reply to /prefix/fixtures instead and return a 404 for /fixtures - this would make things a whole lot easier to debug.

But shipping that change could break existing deployments. Maybe that should be a breaking change for 1.0.

On further thought I'm not going to do this. Having Datasette work behind a proxy the way it does right now is clearly easy for people to deploy (now that I've fixed the bugs) and I trust my improved tests to catch problems in the future.

OK, i think I got all of them this time!

The latest demo is now live at https://datasette-apache-proxy-demo.fly.dev/prefix/fixtures/sortable?_facet=pk2

I'm closing this issue, but feel free to re-open it if you spot any that I missed.

The demo is now live on https://datasette-apache-proxy-demo.fly.dev/prefix/

I'm going to go with Fly instead for this, especially as I can keep it within their free tier (and iI want to get more familiar with their platform).

Here's why it failed - the fly.toml file that was generated when I ran flyctl launch had this section:

[[services]]
  http_checks = []
  internal_port = 8080
  processes = ["app"]
  protocol = "tcp"
  script_checks = []

But I need internal_port to be 80 for Apache, so I changed that and ran flyctl deploy --build-arg DATASETTE_REF=main again - and it worked!

https://floral-dust-4577.fly.dev/prefix/ - not seeing any 503 errors there.

I tried to deploy it to Fly - initially using flyctl launch but then switching to flyctl deploy so I could use the --build-arg option (posted a feature request here).

Almost got it working, but it failed the health check:

% cd datasette/demos/apache-proxy
apache-proxy % flyctl launch
Creating app in /Users/simon/Dropbox/Development/datasette/demos/apache-proxy
Scanning source code
Detected Dockerfile app
Automatically selected personal organization: Simon Willison
? Select region: sjc (Sunnyvale, California (US))
Created app floral-dust-4577 in organization personal
Wrote config file fly.toml
Your app is ready. Deploy with `flyctl deploy`
? Would you like to deploy now? Yes
Deploying floral-dust-4577
==> Validating app configuration
--> Validating app configuration done
Services
TCP 80/443 ⇢ 8080
==> Creating build context
--> Creating build context done
==> Building image with Docker
Sending build context to Docker daemon  8.704kB
...
Error error building: executor failed running [/bin/sh -c pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip]: exit code: 1

# I didn't pass the build argument, trying again with flyctl deploy

apache-proxy % flyctl deploy --build-arg DATASETTE_REF=main 
Update available 0.0.229 -> v0.0.255
Run "flyctl version update" to upgrade
Deploying floral-dust-4577
==> Validating app configuration
--> Validating app configuration done
Services
TCP 80/443 ⇢ 8080
==> Creating build context
--> Creating build context done
==> Building image with Docker
Sending build context to Docker daemon  8.704kB
[+] Building 15.7s (27/27) ...                                                            0.0s
==> Pushing image to fly
The push refers to repository [registry.fly.io/floral-dust-4577]
9bf88c92aa2a: Pushed 
3d61728b8391: Pushed 
...
--> Pushing image done
Image: registry.fly.io/floral-dust-4577:deployment-1637429501
Image size: 276 MB
==> Creating release
Release v2 created

You can detach the terminal anytime without stopping the deployment
Monitoring Deployment

1 desired, 1 placed, 0 healthy, 0 unhealthy [health checks: 1 total, 1 critical]


1 desired, 1 placed, 0 healthy, 1 unhealthy [health checks: 1 total, 1 critical]
v0 failed - Failed due to unhealthy allocations - no stable job version to auto revert to
Failed Instances

==> Failure #1

Instance
  ID            = 36adac86             
  Version       = 0                    
  Region        = sjc                  
  Desired       = run                  
  Status        = running              
  Health Checks = 1 total, 1 critical  
  Restarts      = 0                    
  Created       = 4m52s ago            

Recent Events
TIMESTAMP            TYPE       MESSAGE                 
2021-11-20T17:32:52Z Received   Task received by client 
2021-11-20T17:32:52Z Task Setup Building Task Directory 
2021-11-20T17:33:02Z Started    Task started by client  

Recent Logs
2021-11-20T17:32:56Z [info] Unpacking image
2021-11-20T17:33:01Z [info] Preparing kernel init
2021-11-20T17:33:01Z [info] Configuring firecracker
2021-11-20T17:33:02Z [info] Starting virtual machine
2021-11-20T17:33:02Z [info] Starting init (commit: 7943db6)...
2021-11-20T17:33:02Z [info] Preparing to run: `/usr/bin/supervisord -c /app/supervisord.conf` as root
2021-11-20T17:33:02Z [info] 2021/11/20 17:33:02 listening on [fdaa:0:4ef:a7b:2295:36ad:ac86:2]:22 (DNS: [fdaa::3]:53)
2021-11-20T17:33:02Z [info] 2021-11-20 17:33:02,374 CRIT Supervisor is running as root.  Privileges were not dropped because no user is specified in the config file.  If you intend to run as root, you can set user=root in the config file to avoid this message.
2021-11-20T17:33:02Z [info] 2021-11-20 17:33:02,376 INFO supervisord started with pid 510
2021-11-20T17:33:03Z [info] 2021-11-20 17:33:03,379 INFO spawned: 'apache2' with pid 515
2021-11-20T17:33:03Z [info] 2021-11-20 17:33:03,381 INFO spawned: 'datasette' with pid 516
2021-11-20T17:33:05Z [info] 2021-11-20 17:33:05,068 INFO success: apache2 entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)
2021-11-20T17:33:05Z [info] 2021-11-20 17:33:05,068 INFO success: datasette entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)
2021-11-20T17:33:28Z [error] Health check status changed 'warning' => 'critical'
***v0 failed - Failed due to unhealthy allocations - no stable job version to auto revert to and deploying as v1 

As a a sanity check, would it be worth looking at trying to push the multi-process container on another provider of a knative / cloud run / tekton ? I have a somewhat similar use case for a future proejct, so i'm been very grateful to you sharing all the progress in this issue.

That's a great idea. I'll try running on a non-Knative host too (probably Fly - though they actually run containers using Firecracker which ends up being completely different).

Cloud Run are the only Knative host I've used, know of any others aside from Scaleway? They look like they're worth getting familiar with.

I'm going to leave this issue open, tag it as "help wanted" and cross my fingers that someone with Cloud Run deep expertise takes an interest in figuring out what's going wrong here!

As a a sanity check, would it be worth looking at trying to push the multi-process container on another provider of a knative / cloud run / tekton ? I have a somewhat similar use case for a future proejct, so i'm been very grateful to you sharing all the progress in this issue.

As I understand it, Scaleway also offer a very similar offering using what appear to be many similar components that might at least see if it's an issue with more than one knative based FaaS provider

https://www.scaleway.com/en/serverless-containers/
https://developers.scaleway.com/en/products/containers/api/#main-features

I've now tried both Debian and Alpine, and I've tried both tini and supervisord. Each time I get the same result - I get 503 errors for the first dozen or so refreshes of /prefix/ followed by it intermittently working. Absolutely stumped.

I managed to port the whole thing over to Debian - which took a lot of work because their packaged Apache 2 works very differently from the Alpine one.

Once again... I got it working fine on my laptop, but the image deployed to Cloud Run throws 503 errors!

FROM python:3.9.7-slim-bullseye

RUN apt-get update && \
    apt-get install -y apache2 supervisor && \
    apt clean && \
    rm -rf /var/lib/apt && \
    rm -rf /var/lib/dpkg/info/*

# Apache environment, copied from
# https://github.com/ijklim/laravel-benfords-law-app/blob/e9bf385dcaddb62ea466a7b245ab6e4ef708c313/docker/os/Dockerfile
ENV APACHE_DOCUMENT_ROOT=/var/www/html/public
ENV APACHE_RUN_USER www-data
ENV APACHE_RUN_GROUP www-data
ENV APACHE_PID_FILE /var/run/apache2.pid
ENV APACHE_RUN_DIR /var/run/apache2
ENV APACHE_LOCK_DIR /var/lock/apache2
ENV APACHE_LOG_DIR /var/log
RUN ln -sf /dev/stdout /var/log/apache2-access.log
RUN ln -sf /dev/stderr /var/log/apache2-error.log
RUN mkdir -p $APACHE_RUN_DIR $APACHE_LOCK_DIR

RUN a2enmod proxy
RUN a2enmod proxy_http
RUN a2enmod headers

ARG DATASETTE_REF

RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip

# Append this to the end of the default httpd.conf file
RUN echo '\n\
<Directory /app/html/>\n\
    Options Indexes FollowSymLinks\n\
    AllowOverride None\n\
    Require all granted\n\
</Directory>\n\
\n\
<VirtualHost *:80>\n\
    ServerName localhost\n\
    DocumentRoot /app/html\n\
    ProxyPreserveHost On\n\
    ProxyPass /prefix/ http://127.0.0.1:8001/\n\
    Header add X-Proxied-By "Apache2"\n\
</VirtualHost>\n\
' > /etc/apache2/sites-enabled/000-default.conf

WORKDIR /app
RUN mkdir -p /app/html
RUN echo '<a href="/prefix/">Datasette</a>' > /app/html/index.html

ADD https://latest.datasette.io/fixtures.db /app/fixtures.db

EXPOSE 80

RUN echo "[supervisord]" >> /app/supervisord.conf
RUN echo "nodaemon=true" >> /app/supervisord.conf
RUN echo "" >> /app/supervisord.conf
RUN echo "[program:apache2]" >> /app/supervisord.conf
RUN echo "command=apache2 -D FOREGROUND" >> /app/supervisord.conf
RUN echo "" >> /app/supervisord.conf
RUN echo "[program:datasette]" >> /app/supervisord.conf
RUN echo "command=datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001" >> /app/supervisord.conf

CMD ["/usr/bin/supervisord", "-c", "/app/supervisord.conf"]

First I'm going to try using Debian Buster as the base image instead of Alpine.

Based on https://medium.com/google-cloud/init-process-for-containers-d03a471fa0cc I might try s6.

Aha! This could be the clue I was looking for: https://www.reddit.com/r/googlecloud/comments/fmkx63/comment/fl5csty/?utm_source=reddit&utm_medium=web2x&context=3

Are you processing on a background thread in your container? If so, it's likely your problem, because cloud run will put your app into a low power state between http requests. For long running tasks in cloud run, you need to keep the http connection open, and not return until you are done.

Maybe the datasette & process is being affected by that in some way?

So frustrating, that's giving me the same problem after being deployed! 503 errors for the first while, then it starts working.

OK, that works on my laptop - and Ctrl+C quits it, which is nice:

apache-proxy % docker run -p 5000:80 --rm datasette-build-arg-demo
2021-11-20 02:22:13,925 CRIT Supervisor is running as root.  Privileges were not dropped because no user is specified in the config file.  If you intend to run as root, you can set user=root in the config file to avoid this message.
2021-11-20 02:22:13,927 INFO supervisord started with pid 1
2021-11-20 02:22:14,931 INFO spawned: 'datasette' with pid 7
2021-11-20 02:22:14,934 INFO spawned: 'httpd' with pid 8
2021-11-20 02:22:16,484 INFO success: datasette entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)
2021-11-20 02:22:16,484 INFO success: httpd entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)
^C
2021-11-20 02:22:26,285 WARN received SIGINT indicating exit request
2021-11-20 02:22:26,286 INFO waiting for datasette, httpd to die
2021-11-20 02:22:26,315 INFO stopped: httpd (exit status 0)
2021-11-20 02:22:26,540 INFO stopped: datasette (exit status 0)

Here's my new Dockerfile:

FROM python:3-alpine

RUN apk add --no-cache \
    apache2 \
    apache2-proxy \
    supervisor \
    bash

ARG DATASETTE_REF

RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip

# Append this to the end of the default httpd.conf file
RUN echo -e 'ServerName localhost\n\
\n\
<Proxy *>\n\
  Order deny,allow\n\
  Allow from all\n\
</Proxy>\n\
\n\
ProxyPreserveHost On\n\
ProxyPass /prefix/ http://127.0.0.1:8001/\n\
Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf

RUN echo '<a href="/prefix/">Datasette</a>' > /var/www/localhost/htdocs/index.html

WORKDIR /app

ADD https://latest.datasette.io/fixtures.db /app/fixtures.db

EXPOSE 80

RUN echo "[supervisord]" >> /app/supervisord.conf
RUN echo "nodaemon=true" >> /app/supervisord.conf
RUN echo "" >> /app/supervisord.conf
RUN echo "[program:httpd]" >> /app/supervisord.conf
RUN echo "command=httpd -D FOREGROUND" >> /app/supervisord.conf
RUN echo "" >> /app/supervisord.conf
RUN echo "[program:datasette]" >> /app/supervisord.conf
RUN echo "command=datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001" >> /app/supervisord.conf

CMD ["/usr/bin/supervisord", "-c", "/app/supervisord.conf"]

https://docs.docker.com/config/containers/multi-service_container/ suggests supervisord as a last resort.

https://stackoverflow.com/a/49100302/6083 has a neat looking recipe for than in Alpine:

1. Dockerfile is:

FROM alpine:latest
RUN apk update && apk add --no-cache supervisor openssh nginx
COPY supervisord.conf /etc/supervisord.conf
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]

2. supervisord.conf is:

[supervisord]
nodaemon=true

[program:sshd]
command=/usr/sbin/sshd -D

[program:nginx]
command=nginx -c /etc/nginx/nginx.conf

Again, that approach worked on my laptop but when deployed to Cloud Run mostly gave me 503 errors for the /prefix/ page, with the occasional 200.

I did this:

RUN echo "#!/bin/bash" >> start.sh
# Start Datasette running in background with &
RUN echo "datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001 &" >> /app/start.sh
RUN echo "httpd -D FOREGROUND" >> /app/start.sh

RUN chmod +x /app/start.sh

CMD /app/start.sh

I'm going to try running Apache with httpd -D FOREGROUND while running datasette &.

Waiting for health check to begin makes it sound like the container didn't start properly.

That eventually failed, but I did get these in the build logs:

https://user-images.githubusercontent.com/9599/142710753-323566ed-cd1e-4492-ac8e-dae85c8bd6f9.png">

Deploy to Cloud Run appears to hang here:

Deploying container to Cloud Run service [datasette-apache-proxy-demo] in project [datasette-222320] region [us-central1]
⠧ Deploying... Revision deployment finished. Waiting for health check to begin.                                                                               
  ⠧ Creating Revision...                                                                                                                                      
  . Routing traffic...                                                                                                                                        
  ✓ Setting IAM Policy...                                                                                                                                     

Waiting for health check to begin makes it sound like the container didn't start properly.

I figured out a recipe to run httpd as a service inside Alpine - works great on my laptop, here's my new Dockerfile:

FROM python:3-alpine

# openrc gives us rc-service

RUN apk add --no-cache \
    openrc \
    apache2 \
    apache2-proxy \
    bash

ARG DATASETTE_REF

RUN pip install https://github.com/simonw/datasette/archive/${DATASETTE_REF}.zip

# Append this to the end of the default httpd.conf file
RUN echo -e 'ServerName localhost\n\
\n\
<Proxy *>\n\
  Order deny,allow\n\
  Allow from all\n\
</Proxy>\n\
\n\
ProxyPreserveHost On\n\
ProxyPass /prefix/ http://127.0.0.1:8001/\n\
Header add X-Proxied-By "Apache2"' >> /etc/apache2/httpd.conf

RUN echo '<a href="/prefix/">Datasette</a>' > /var/www/localhost/htdocs/index.html

WORKDIR /app

ADD https://latest.datasette.io/fixtures.db /app/fixtures.db

EXPOSE 80

# RUN echo -e "#!/bin/bash\nopenrc default\nrc-service apache2 start;\ndatasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001" > /app/start.sh

RUN echo "#!/bin/bash" >> start.sh
RUN echo "openrc default" >> start.sh
RUN echo "rc-service apache2 start" >> start.sh
RUN echo "datasette /app/fixtures.db --setting base_url '/prefix/' --version-note '${DATASETTE_REF}' -h 0.0.0.0 -p 8001" >> /app/start.sh

RUN chmod +x /app/start.sh

CMD /app/start.sh

I'm going to try this on Cloud Run and see if it fixes the 503s

One annoying thing about this: Ctrl+C on my laptop no longer stops the container, I have to docker ps and then docker kill xxx instead.

I have a hunch that running httpd -D FOREGROUND doesn't show error logs, which would explain why I can't use the Cloud Run logs to figure out the reason for the 503s.

Yup, that fixed it.

https://apache-proxy-demo.datasette.io/prefix/fixtures/compound_three_primary_keys has broken suggested facet links - they go to https://localhost:8001/prefix/fixtures/compound_three_primary_keys?_facet=pk1#facet-pk1 - but I think that's because I'm missing the ProxyPreserveHost On setting.