html_url,issue_url,id,node_id,user,user_label,created_at,updated_at,author_association,body,reactions,issue,issue_label,performed_via_github_app
https://github.com/simonw/datasette/issues/1878#issuecomment-1336100218,https://api.github.com/repos/simonw/datasette/issues/1878,1336100218,IC_kwDOBm6k_c5Po0V6,9599,simonw,2022-12-03T07:02:15Z,2022-12-03T07:02:15Z,OWNER,"Moved this work to a PR:
- #1931","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1432013704,/db/table/-/upsert API,
https://github.com/simonw/datasette/issues/1878#issuecomment-1336094562,https://api.github.com/repos/simonw/datasette/issues/1878,1336094562,IC_kwDOBm6k_c5Poy9i,9599,simonw,2022-12-03T06:27:50Z,2022-12-03T06:29:06Z,OWNER,"This adds it to the API explorer:
```diff
diff --git a/datasette/views/special.py b/datasette/views/special.py
index 1f84b094..1b4a9d3c 100644
--- a/datasette/views/special.py
+++ b/datasette/views/special.py
@@ -316,21 +316,37 @@ class ApiExplorerView(BaseView):
request.actor, ""insert-row"", (name, table)
):
pks = await db.primary_keys(table)
- table_links.append(
- {
- ""path"": self.ds.urls.table(name, table) + ""/-/insert"",
- ""method"": ""POST"",
- ""label"": ""Insert rows into {}"".format(table),
- ""json"": {
- ""rows"": [
- {
- column: None
- for column in await db.table_columns(table)
- if column not in pks
- }
- ]
+ table_links.extend(
+ [
+ {
+ ""path"": self.ds.urls.table(name, table) + ""/-/insert"",
+ ""method"": ""POST"",
+ ""label"": ""Insert rows into {}"".format(table),
+ ""json"": {
+ ""rows"": [
+ {
+ column: None
+ for column in await db.table_columns(table)
+ if column not in pks
+ }
+ ]
+ },
},
- }
+ {
+ ""path"": self.ds.urls.table(name, table) + ""/-/upsert"",
+ ""method"": ""POST"",
+ ""label"": ""Upsert rows into {}"".format(table),
+ ""json"": {
+ ""rows"": [
+ {
+ column: None
+ for column in await db.table_columns(table)
+ if column not in pks
+ }
+ ]
+ },
+ },
+ ]
)
if await self.ds.permission_allowed(
request.actor, ""drop-table"", (name, table)
```
Except it doesn't quite, because the example JSON this generates is invalid as it does not include the primary key column.
(Made me notice that the way example columns are created for `/-/insert` will fail for tables that don't have an auto-incrementing primary key)","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1432013704,/db/table/-/upsert API,
https://github.com/simonw/datasette/issues/1878#issuecomment-1336094470,https://api.github.com/repos/simonw/datasette/issues/1878,1336094470,IC_kwDOBm6k_c5Poy8G,9599,simonw,2022-12-03T06:27:13Z,2022-12-03T06:27:13Z,OWNER,"Tests are going to need to cover both rowid-only and compound primary key tables, including all of the error states.","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1432013704,/db/table/-/upsert API,
https://github.com/simonw/datasette/issues/1878#issuecomment-1336094381,https://api.github.com/repos/simonw/datasette/issues/1878,1336094381,IC_kwDOBm6k_c5Poy6t,9599,simonw,2022-12-03T06:26:25Z,2022-12-03T06:26:25Z,OWNER,"Initial prototype:
```diff
diff --git a/datasette/app.py b/datasette/app.py
index 125b4969..282c0984 100644
--- a/datasette/app.py
+++ b/datasette/app.py
@@ -40,7 +40,7 @@ from .views.special import (
PermissionsDebugView,
MessagesDebugView,
)
-from .views.table import TableView, TableInsertView, TableDropView
+from .views.table import TableView, TableInsertView, TableUpsertView, TableDropView
from .views.row import RowView, RowDeleteView, RowUpdateView
from .renderer import json_renderer
from .url_builder import Urls
@@ -1292,6 +1292,10 @@ class Datasette:
TableInsertView.as_view(self),
r""/(?P[^\/\.]+)/(?P[^\/\.]+)/-/insert$"",
)
+ add_route(
+ TableUpsertView.as_view(self),
+ r""/(?P[^\/\.]+)/(?P[^\/\.]+)/-/upsert$"",
+ )
add_route(
TableDropView.as_view(self),
r""/(?P[^\/\.]+)/(?P[^\/\.]+)/-/drop$"",
diff --git a/datasette/views/table.py b/datasette/views/table.py
index 7ba78c11..ae0d6366 100644
--- a/datasette/views/table.py
+++ b/datasette/views/table.py
@@ -1074,9 +1074,15 @@ class TableInsertView(BaseView):
def __init__(self, datasette):
self.ds = datasette
- async def _validate_data(self, request, db, table_name):
+ async def _validate_data(self, request, db, table_name, pks, upsert):
errors = []
+ pks_list = []
+ if isinstance(pks, str):
+ pks_list = [pks]
+ else:
+ pks_list = list(pks)
+
def _errors(errors):
return None, errors, {}
@@ -1135,6 +1141,15 @@ class TableInsertView(BaseView):
# Validate columns of each row
columns = set(await db.table_columns(table_name))
for i, row in enumerate(rows):
+ if upsert:
+ # It MUST have the primary key
+ missing_pks = [pk for pk in pks_list if pk not in row]
+ if missing_pks:
+ errors.append(
+ 'Row {} is missing primary key column(s): ""{}""'.format(
+ i, '"", ""'.join(missing_pks)
+ )
+ )
invalid_columns = set(row.keys()) - columns
if invalid_columns:
errors.append(
@@ -1146,7 +1161,7 @@ class TableInsertView(BaseView):
return _errors(errors)
return rows, errors, extras
- async def post(self, request):
+ async def post(self, request, upsert=False):
try:
resolved = await self.ds.resolve_table(request)
except NotFound as e:
@@ -1164,7 +1179,12 @@ class TableInsertView(BaseView):
request.actor, ""insert-row"", resource=(database_name, table_name)
):
return _error([""Permission denied""], 403)
- rows, errors, extras = await self._validate_data(request, db, table_name)
+
+ pks = await db.primary_keys(table_name)
+
+ rows, errors, extras = await self._validate_data(
+ request, db, table_name, pks, upsert
+ )
if errors:
return _error(errors, 400)
@@ -1172,15 +1192,19 @@ class TableInsertView(BaseView):
replace = extras.get(""replace"")
should_return = bool(extras.get(""return"", False))
- # Insert rows
- def insert_rows(conn):
+
+ def insert_or_upsert_rows(conn):
table = sqlite_utils.Database(conn)[table_name]
+ kwargs = {}
+ if upsert:
+ kwargs[""pk""] = pks[0] if len(pks) == 1 else pks
+ else:
+ kwargs = {""ignore"": ignore, ""replace"": replace}
if should_return:
rowids = []
+ method = table.upsert if upsert else table.insert
for row in rows:
- rowids.append(
- table.insert(row, ignore=ignore, replace=replace).last_rowid
- )
+ rowids.append(method(row, **kwargs).last_rowid)
return list(
table.rows_where(
""rowid in ({})"".format("","".join(""?"" for _ in rowids)),
@@ -1188,10 +1212,11 @@ class TableInsertView(BaseView):
)
)
else:
- table.insert_all(rows, ignore=ignore, replace=replace)
+ method_all = table.upsert_all if upsert else table.insert_all
+ method_all(rows, **kwargs)
try:
- rows = await db.execute_write_fn(insert_rows)
+ rows = await db.execute_write_fn(insert_or_upsert_rows)
except Exception as e:
return _error([str(e)])
result = {""ok"": True}
@@ -1200,6 +1225,13 @@ class TableInsertView(BaseView):
return Response.json(result, status=201)
+class TableUpsertView(TableInsertView):
+ name = ""table-upsert""
+
+ async def post(self, request):
+ return await super().post(request, upsert=True)
+
+
class TableDropView(BaseView):
name = ""table-drop""
```
Manual testing reveals that this mostly works... but it's not doing the right thing for `""return"": true` - it always returns an empty list.","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1432013704,/db/table/-/upsert API,
https://github.com/simonw/datasette/issues/1878#issuecomment-1336073212,https://api.github.com/repos/simonw/datasette/issues/1878,1336073212,IC_kwDOBm6k_c5Potv8,9599,simonw,2022-12-03T05:38:49Z,2022-12-03T05:38:49Z,OWNER,And on Discord today: https://discord.com/channels/823971286308356157/823971286941302908/1048426072066236536,"{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1432013704,/db/table/-/upsert API,
https://github.com/simonw/datasette/issues/1878#issuecomment-1336070843,https://api.github.com/repos/simonw/datasette/issues/1878,1336070843,IC_kwDOBm6k_c5PotK7,9599,simonw,2022-12-03T05:37:53Z,2022-12-03T05:37:53Z,OWNER,Also requested here: https://news.ycombinator.com/item?id=33839894,"{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1432013704,/db/table/-/upsert API,
https://github.com/simonw/datasette/issues/1878#issuecomment-1312534826,https://api.github.com/repos/simonw/datasette/issues/1878,1312534826,IC_kwDOBm6k_c5OO7Eq,18738650,stevecrawshaw,2022-11-12T17:34:58Z,2022-11-12T17:34:58Z,NONE,"Hi Simon. I have just started experimenting with datasette in earnest, looking at it's suitability for air quality open data. A bulk upsert \ upsert_all would be very useful for me in enabling real time data to be pushed from a sql server database with FME server to a datasette db.
An hourly process queries the last 2 hours of data and pushes that to my database, inserting new data and updating existing combinations of pk siteid and date_time. This is already implemented on our current [open data portal](https://opendata.bristol.gov.uk/explore/dataset/air-quality-data-continuous/table/?disjunctive.location&sort=date_time). Excited to see your progress with this! Thank you for this amazing software.","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1432013704,/db/table/-/upsert API,
https://github.com/simonw/datasette/issues/1878#issuecomment-1299071456,https://api.github.com/repos/simonw/datasette/issues/1878,1299071456,IC_kwDOBm6k_c5NbkHg,9599,simonw,2022-11-01T20:02:43Z,2022-11-01T20:02:43Z,OWNER,"Note that ""update"" is partially covered by the `replace` option to `/-/insert`, added here:
- https://github.com/simonw/datasette/issues/1873#issuecomment-1298885451
","{""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",1432013704,/db/table/-/upsert API,