id,node_id,number,title,user,state,locked,assignee,milestone,comments,created_at,updated_at,closed_at,author_association,pull_request,body,repo,type,active_lock_reason,performed_via_github_app,reactions,draft,state_reason 411066700,MDU6SXNzdWU0MTEwNjY3MDA=,10,Error in upsert if column named 'order',82988,closed,0,,,1,2019-02-16T12:05:18Z,2019-02-24T16:55:38Z,2019-02-24T16:55:37Z,NONE,,"The following works fine: ``` connX = sqlite3.connect('DELME.db', timeout=10) dfX=pd.DataFrame({'col1':range(3),'col2':range(3)}) DBX = Database(connX) DBX['test'].upsert_all(dfX.to_dict(orient='records')) ``` But if a column is named `order`: ``` connX = sqlite3.connect('DELME.db', timeout=10) dfX=pd.DataFrame({'order':range(3),'col2':range(3)}) DBX = Database(connX) DBX['test'].upsert_all(dfX.to_dict(orient='records')) ``` it throws an error: ``` --------------------------------------------------------------------------- OperationalError Traceback (most recent call last) in 3 dfX=pd.DataFrame({'order':range(3),'col2':range(3)}) 4 DBX = Database(connX) ----> 5 DBX['test'].upsert_all(dfX.to_dict(orient='records')) /usr/local/lib/python3.7/site-packages/sqlite_utils/db.py in upsert_all(self, records, pk, foreign_keys, column_order) 347 foreign_keys=foreign_keys, 348 upsert=True, --> 349 column_order=column_order, 350 ) 351 /usr/local/lib/python3.7/site-packages/sqlite_utils/db.py in insert_all(self, records, pk, foreign_keys, upsert, batch_size, column_order) 327 jsonify_if_needed(record.get(key, None)) for key in all_columns 328 ) --> 329 result = self.db.conn.execute(sql, values) 330 self.db.conn.commit() 331 self.last_id = result.lastrowid OperationalError: near ""order"": syntax error ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/10/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 601358649,MDU6SXNzdWU2MDEzNTg2NDk=,100,"Mechanism for forcing column-type, over-riding auto-detection",9599,closed,0,,,3,2020-04-16T19:12:52Z,2020-04-17T23:53:32Z,2020-04-17T23:53:32Z,OWNER,,"As seen in https://github.com/dogsheep/github-to-sqlite/issues/27#issuecomment-614843406 - there's a problem where you insert a record with a `None` value for a column and that column is created as `TEXT` - but actually you intended it to be an `INT` (as later examples will demonstrate). Some kind of mechanism for over-riding the detected types of columns would be useful here.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/100/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 601392318,MDU6SXNzdWU2MDEzOTIzMTg=,101,README should include an example of CLI data insertion,9599,closed,0,,,0,2020-04-16T19:45:37Z,2020-04-17T23:59:49Z,2020-04-17T23:59:49Z,OWNER,,Maybe using `curl` from the GitHub API.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/101/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 602569315,MDU6SXNzdWU2MDI1NjkzMTU=,102,Can't store an array or dictionary containing a bytes value,9599,closed,0,,,0,2020-04-18T22:49:21Z,2020-05-01T20:45:45Z,2020-05-01T20:45:45Z,OWNER,,"``` In [1]: import sqlite_utils In [2]: db = sqlite_utils.Database(memory=True) In [3]: db[""t""].insert({""id"": 1, ""data"": {""foo"": b""bytes""}}) --------------------------------------------------------------------------- TypeError Traceback (most recent call last) in ----> 1 db[""t""].insert({""id"": 1, ""data"": {""foo"": b""bytes""}}) ~/Dropbox/Development/sqlite-utils/sqlite_utils/db.py in insert(self, record, pk, foreign_keys, column_order, not_null, defaults, hash_id, alter, ignore, replace, extracts, conversions, columns) 950 extracts=extracts, 951 conversions=conversions, --> 952 columns=columns, 953 ) 954 ~/Dropbox/Development/sqlite-utils/sqlite_utils/db.py in insert_all(self, records, pk, foreign_keys, column_order, not_null, defaults, batch_size, hash_id, alter, ignore, replace, extracts, conversions, columns, upsert) 1052 for key in all_columns: 1053 value = jsonify_if_needed( -> 1054 record.get(key, None if key != hash_id else _hash(record)) 1055 ) 1056 if key in extracts: ~/Dropbox/Development/sqlite-utils/sqlite_utils/db.py in jsonify_if_needed(value) 1318 def jsonify_if_needed(value): 1319 if isinstance(value, (dict, list, tuple)): -> 1320 return json.dumps(value) 1321 elif isinstance(value, (datetime.time, datetime.date, datetime.datetime)): 1322 return value.isoformat() /usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/__init__.py in dumps(obj, skipkeys, ensure_ascii, check_circular, allow_nan, cls, indent, separators, default, sort_keys, **kw) 229 cls is None and indent is None and separators is None and 230 default is None and not sort_keys and not kw): --> 231 return _default_encoder.encode(obj) 232 if cls is None: 233 cls = JSONEncoder /usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py in encode(self, o) 197 # exceptions aren't as detailed. The list call should be roughly 198 # equivalent to the PySequence_Fast that ''.join() would do. --> 199 chunks = self.iterencode(o, _one_shot=True) 200 if not isinstance(chunks, (list, tuple)): 201 chunks = list(chunks) /usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py in iterencode(self, o, _one_shot) 255 self.key_separator, self.item_separator, self.sort_keys, 256 self.skipkeys, _one_shot) --> 257 return _iterencode(o, 0) 258 259 def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, /usr/local/Cellar/python/3.7.4_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/json/encoder.py in default(self, o) 177 178 """""" --> 179 raise TypeError(f'Object of type {o.__class__.__name__} ' 180 f'is not JSON serializable') 181 TypeError: Object of type bytes is not JSON serializable ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/102/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 610517472,MDU6SXNzdWU2MTA1MTc0NzI=,103,sqlite3.OperationalError: too many SQL variables in insert_all when using rows with varying numbers of columns,32605365,closed,0,,,8,2020-05-01T02:26:14Z,2020-05-14T00:18:57Z,2020-05-14T00:18:57Z,CONTRIBUTOR,,"If using insert_all to put in 1000 rows of data with varying number of columns, it comes up with this message `sqlite3.OperationalError: too many SQL variables` if the number of columns is larger in later records (past the first row) I've reduced `SQLITE_MAX_VARS` by 100 to 899 at the top of `db.py` to add wiggle room, so that if the column count increases it wont go past SQLite's batch limit as calculated by this line of code based on the count of the first row's dict keys batch_size = max(1, min(batch_size, SQLITE_MAX_VARS // num_columns))",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/103/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 610853393,MDU6SXNzdWU2MTA4NTMzOTM=,104,"--schema option to ""sqlite-utils tables""",9599,closed,0,,,0,2020-05-01T16:55:49Z,2020-05-01T17:12:37Z,2020-05-01T17:12:37Z,OWNER,,Adds output showing the table schema.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/104/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 610853576,MDU6SXNzdWU2MTA4NTM1NzY=,105,"""sqlite-utils views"" command",9599,closed,0,,,1,2020-05-01T16:56:11Z,2020-05-01T20:40:07Z,2020-05-01T20:38:36Z,OWNER,,Similar to `sqlite-utils tables`. See also #104.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/105/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 611216862,MDU6SXNzdWU2MTEyMTY4NjI=,106,"create_view(..., ignore=True, replace=True) parameters",9599,closed,0,,,1,2020-05-02T15:45:21Z,2020-05-02T16:04:51Z,2020-05-02T16:02:10Z,OWNER,,"Two new parameters which specify what should happen if the view already exists. I want this for https://github.com/dogsheep/github-to-sqlite/issues/37 Here's the current `create_view()` implementation: https://github.com/simonw/sqlite-utils/blob/b4d953d3ccef28bb81cea40ca165a647b59971fa/sqlite_utils/db.py#L325-L332 `ignore=True` will not do anything if the view exists already. `replace=True` will drop and redefine the view - but only if its SQL definition differs, otherwise it will be left alone.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/106/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 611222968,MDU6SXNzdWU2MTEyMjI5Njg=,107,sqlite-utils create-view CLI command,9599,closed,0,,,2,2020-05-02T16:15:13Z,2020-05-03T15:36:58Z,2020-05-03T15:36:37Z,OWNER,,Can go with #27 - `sqlite-utils create-table`.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/107/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 611326701,MDU6SXNzdWU2MTEzMjY3MDE=,108,Documentation unit tests for CLI commands,9599,closed,0,,,2,2020-05-03T03:58:42Z,2020-05-03T04:13:57Z,2020-05-03T04:13:57Z,OWNER,,Have a test that ensures all CLI commands are documented.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/108/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 612658444,MDU6SXNzdWU2MTI2NTg0NDQ=,109,"table.create_index(..., ignore=True)",9599,closed,0,,,1,2020-05-05T14:44:21Z,2020-05-05T14:46:53Z,2020-05-05T14:46:53Z,OWNER,,Option to silently do nothing if the index already exists.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/109/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 413740684,MDU6SXNzdWU0MTM3NDA2ODQ=,11,Detect numpy types when creating tables,9599,closed,0,,,2,2019-02-23T21:09:35Z,2019-02-24T04:02:20Z,2019-02-24T04:02:20Z,OWNER,,Inspired by #8,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/11/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 613755043,MDU6SXNzdWU2MTM3NTUwNDM=,110,Support decimal.Decimal type,134771,closed,0,,,6,2020-05-07T03:57:19Z,2020-05-11T01:58:20Z,2020-05-11T01:50:11Z,NONE,,"Decimal types in Postgres cause a failure in db.py data type selection --- I have a Django app using a MoneyField, which uses a `numeric(14,0)` data type in Postgres (https://www.postgresql.org/docs/9.3/datatype-numeric.html). When attempting to export that table I get the following error: ```bash $ db-to-sqlite --table isaweb_proposal ""postgres://connection"" test.db .... column_type=COLUMN_TYPE_MAPPING[column_type], KeyError: ``` Looking at `sql_utils.db.py` at 292-ish it's clear that there is no matching type for what I assume SQLAlchemy interprets as Python decimal.Decimal. From the [SQLite docs](https://www.sqlite.org/datatype3.html#affinity_name_examples) it looks like DECIMAL in other DBs are considered numeric. I'm not quite sure if it's as simple as adding a data type to that list or if there are repercussions beyond it. Thanks for a great tool!",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/110/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 615477131,MDU6SXNzdWU2MTU0NzcxMzE=,111,sqlite-utils drop-table and drop-view commands,9599,closed,0,,,2,2020-05-10T21:10:42Z,2020-05-11T01:58:36Z,2020-05-11T00:44:26Z,OWNER,,Would be useful to be able to drop views and tables from the CLI.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/111/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 616271236,MDU6SXNzdWU2MTYyNzEyMzY=,112,"add_foreign_key(...., ignore=True)",9599,closed,0,,5896742,4,2020-05-12T00:24:00Z,2020-09-20T22:17:34Z,2020-09-20T22:17:34Z,OWNER,,"When using this library I often find myself wanting to ""add this foreign key, but only if it doesn't exist yet"". The `ignore=True` parameter is increasingly being used for this else where in the library (e.g. in `create_view()`).",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/112/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 621286870,MDU6SXNzdWU2MjEyODY4NzA=,113,Syntactic sugar for ATTACH DATABASE,9599,closed,0,,,2,2020-05-19T21:10:00Z,2021-02-19T05:09:12Z,2021-02-19T04:56:36Z,OWNER,,"https://www.sqlite.org/lang_attach.html Maybe something like this: ```python db.attach(""other_db"", ""other_db.db"") ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/113/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 621989740,MDU6SXNzdWU2MjE5ODk3NDA=,114,table.transform() method for advanced alter table,9599,closed,0,,5897911,26,2020-05-20T18:20:46Z,2020-09-22T07:51:37Z,2020-09-22T04:20:02Z,OWNER,,"SQLite's `ALTER TABLE` can only do the following: * Rename a table * Rename a column * Add a column Notably, it cannot drop columns - so tricks like ""add a float version of this text column, populate it, then drop the old one and rename"" won't work. The docs here https://www.sqlite.org/lang_altertable.html#making_other_kinds_of_table_schema_changes describe a way of implementing full alters safely within a transaction, but it's fiddly. 1. Create new table 2. Copy data 3. Drop old table 4. Rename new into old It would be great if `sqlite-utils` provided an abstraction to help make these kinds of changes safely.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/114/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 637889964,MDU6SXNzdWU2Mzc4ODk5NjQ=,115,Ability to execute insert/update statements with the CLI,9599,closed,0,,,1,2020-06-12T17:01:17Z,2020-06-12T17:51:11Z,2020-06-12T17:41:10Z,OWNER,,"``` $ sqlite-utils github.db ""update stars set starred_at = ''"" Traceback (most recent call last): File ""/Users/simon/.local/bin/sqlite-utils"", line 8, in sys.exit(cli()) File ""/Users/simon/.local/pipx/venvs/sqlite-utils/lib/python3.8/site-packages/click/core.py"", line 829, in __call__ return self.main(*args, **kwargs) File ""/Users/simon/.local/pipx/venvs/sqlite-utils/lib/python3.8/site-packages/click/core.py"", line 782, in main rv = self.invoke(ctx) File ""/Users/simon/.local/pipx/venvs/sqlite-utils/lib/python3.8/site-packages/click/core.py"", line 1259, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File ""/Users/simon/.local/pipx/venvs/sqlite-utils/lib/python3.8/site-packages/click/core.py"", line 1066, in invoke return ctx.invoke(self.callback, **ctx.params) File ""/Users/simon/.local/pipx/venvs/sqlite-utils/lib/python3.8/site-packages/click/core.py"", line 610, in invoke return callback(*args, **kwargs) File ""/Users/simon/.local/pipx/venvs/sqlite-utils/lib/python3.8/site-packages/sqlite_utils/cli.py"", line 673, in query headers = [c[0] for c in cursor.description] TypeError: 'NoneType' object is not iterable ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/115/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 644122661,MDU6SXNzdWU2NDQxMjI2NjE=,116,Documentation for table.pks introspection property,9599,closed,0,,,2,2020-06-23T20:27:24Z,2020-06-23T21:21:33Z,2020-06-23T21:03:14Z,OWNER,,https://github.com/simonw/sqlite-utils/blob/4d9a3204361d956440307a57bd18c829a15861db/sqlite_utils/db.py#L535-L540,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/116/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 644161221,MDU6SXNzdWU2NDQxNjEyMjE=,117,Support for compound (composite) foreign keys,9599,open,0,,,3,2020-06-23T21:33:42Z,2020-06-23T21:40:31Z,,OWNER,,"It turns out SQLite supports composite foreign keys: https://www.sqlite.org/foreignkeys.html#fk_composite Their example looks like this: ```sql CREATE TABLE album( albumartist TEXT, albumname TEXT, albumcover BINARY, PRIMARY KEY(albumartist, albumname) ); CREATE TABLE song( songid INTEGER, songartist TEXT, songalbum TEXT, songname TEXT, FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) ); ``` Here's what that looks like in sqlite-utils: ``` In [1]: import sqlite_utils In [2]: import sqlite3 In [3]: conn = sqlite3.connect("":memory:"") In [4]: conn Out[4]: In [5]: conn.executescript("""""" ...: CREATE TABLE album( ...: albumartist TEXT, ...: albumname TEXT, ...: albumcover BINARY, ...: PRIMARY KEY(albumartist, albumname) ...: ); ...: ...: CREATE TABLE song( ...: songid INTEGER, ...: songartist TEXT, ...: songalbum TEXT, ...: songname TEXT, ...: FOREIGN KEY(songartist, songalbum) REFERENCES album(albumartist, albumname) ...: ); ...: """""") Out[5]: In [6]: db = sqlite_utils.Database(conn) In [7]: db.tables Out[7]: [,
] In [8]: db.tables[0].foreign_keys Out[8]: [] In [9]: db.tables[1].foreign_keys Out[9]: [ForeignKey(table='song', column='songartist', other_table='album', other_column='albumartist'), ForeignKey(table='song', column='songalbum', other_table='album', other_column='albumname')] ``` The table appears to have two separate foreign keys, when actually it has a single compound composite foreign key.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/117/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 652700770,MDU6SXNzdWU2NTI3MDA3NzA=,119,Ability to remove a foreign key,9599,closed,0,,,3,2020-07-07T22:31:37Z,2020-09-24T20:36:59Z,2020-09-24T20:36:59Z,OWNER,,Useful if you add one but make a mistake and need to undo it without recreating the database from scratch.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/119/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 652961907,MDU6SXNzdWU2NTI5NjE5MDc=,121,Improved (and better documented) support for transactions,9599,open,0,,,3,2020-07-08T04:56:51Z,2020-09-24T20:36:46Z,,OWNER,,"_Originally posted by @simonw in https://github.com/simonw/sqlite-utils/pull/118#issuecomment-655283393_ We should put some thought into how this library supports and encourages smart use of transactions.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/121/reactions"", ""total_count"": 1, ""+1"": 1, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 665700495,MDU6SXNzdWU2NjU3MDA0OTU=,122,CLI utility for inserting binary files into SQLite,9599,closed,0,,,10,2020-07-26T03:27:39Z,2020-07-27T07:10:41Z,2020-07-27T07:09:03Z,OWNER,,"SQLite BLOB columns can store entire binary files. The challenge is inserting them, since they don't neatly fit into JSON objects. It would be great if the `sqlite-utils` CLI had a trick for helping with this. Inspired by https://github.com/simonw/datasette-media/issues/14",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/122/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 665701216,MDU6SXNzdWU2NjU3MDEyMTY=,123,--raw option for outputting binary content,9599,closed,0,,,0,2020-07-26T03:35:39Z,2020-07-26T16:44:11Z,2020-07-26T16:44:11Z,OWNER,,"Related to the `insert-files` work in #122 - it should be easy to get binary data back out of the database again. One way to do that could be: sqlite-utils files.db ""select content from files where key = 'foo.jpg'"" --raw The `--raw` option would cause just the contents of the first column to be output directly to stdout.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/123/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 665802405,MDU6SXNzdWU2NjU4MDI0MDU=,124,sqlite-utils query should support named parameters,9599,closed,0,,,1,2020-07-26T15:25:10Z,2020-07-30T22:57:51Z,2020-07-27T03:53:58Z,OWNER,,"To help out with escaping - so you can run this: sqlite-utils query ""insert into foo (blah) values (:blah)"" --param blah `something here`",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/124/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 665817570,MDU6SXNzdWU2NjU4MTc1NzA=,125,"Output binary columns in ""sqlite-utils query"" JSON",9599,closed,0,,,4,2020-07-26T16:47:02Z,2020-07-27T00:49:41Z,2020-07-27T00:48:45Z,OWNER,,You get an error if you try to run a query that returns data from a BLOB.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/125/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 665819048,MDU6SXNzdWU2NjU4MTkwNDg=,126,Ability to insert binary data on the CLI using JSON,9599,closed,0,,,2,2020-07-26T16:54:14Z,2020-07-27T04:00:33Z,2020-07-27T03:59:45Z,OWNER,,"> I could solve round tripping (at least a bit) by allowing insert to be run with a flag that says ""these columns are base64 encoded, store the decoded data in a BLOB"". > > That would solve inserting binary data using JSON too. _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/issues/125#issuecomment-664012247_",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/126/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 666040390,MDU6SXNzdWU2NjYwNDAzOTA=,127,Ability to insert files piped to insert-files stdin,9599,closed,0,,,3,2020-07-27T07:09:33Z,2020-07-30T03:08:52Z,2020-07-30T03:08:18Z,OWNER,,"> Inserting files by piping them in should work - but since a filename cannot be derived this will need a `--name blah.gif` option. > > cat blah.gif | sqlite-utils insert-files files.db files - --name=blah.gif > _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/issues/122#issuecomment-664128071_",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/127/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 666639051,MDU6SXNzdWU2NjY2MzkwNTE=,128,Support UUID and memoryview types,9599,closed,0,,,1,2020-07-27T23:08:34Z,2020-07-30T01:10:43Z,2020-07-30T01:10:43Z,OWNER,,`psycopg2` can return data from PostgreSQL as `uuid.UUID` or `memoryview` objects. These should to be supported by `sqlite-utils` - mainly for https://github.com/simonw/db-to-sqlite,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/128/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 668308777,MDU6SXNzdWU2NjgzMDg3Nzc=,129,"""insert-files --sqlar"" for creating SQLite archives",9599,closed,0,,,2,2020-07-30T02:28:29Z,2020-07-30T22:41:01Z,2020-07-30T22:40:55Z,OWNER,,"A `--sqlar` option could cause `insert-files` to behave in the same way as SQLite's own sqlar mechanism. https://www.sqlite.org/sqlar.html and https://sqlite.org/sqlar/doc/trunk/README.md",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/129/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 413779210,MDU6SXNzdWU0MTM3NzkyMTA=,13,Ability to automatically create IDs from content hash of row,9599,closed,0,,,1,2019-02-24T04:07:08Z,2019-02-24T04:36:48Z,2019-02-24T04:36:48Z,OWNER,,"Sometimes when you are importing data the underlying source provides records without IDs that can be uniquely identified by their contents. A utility mechanism for calculating a sha1 hash of the contents and using that as a unique ID would be useful.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/13/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 671130371,MDU6SXNzdWU2NzExMzAzNzE=,130,Support tokenize option for FTS,9599,closed,0,,,3,2020-08-01T19:27:22Z,2020-08-01T20:51:28Z,2020-08-01T20:51:14Z,OWNER,,"FTS5 supports things like porter stemming using a `tokenize=` option: https://www.sqlite.org/fts5.html#tokenizers Something like this in code: ``` CREATE VIRTUAL TABLE [{table}_fts] USING {fts_version} ( {columns}, tokenize='porter', content=[{table}] ); ``` I tried this out just now and it worked exactly as expected. So... `db[table].enable_fts(...) should accept a 'tokenize=` argument, and `sqlite-utils enable-fts ...` should support a `--tokenize` option.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/130/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 675753042,MDU6SXNzdWU2NzU3NTMwNDI=,131,sqlite-utils insert: options for column types,9599,open,0,,,5,2020-08-09T18:59:11Z,2022-03-15T13:21:42Z,,OWNER,,"The `insert` command currently results in string types for every column - at least when used against CSV or TSV inputs. It would be useful if you could do the following: - automatically detects the column types based on eg the first 1000 records - explicitly state the rule for specific columns `--detect-types` could work for the former - or it could do that by default and allow opt-out using `--no-detect-types` For specific columns maybe this: sqlite-utils insert db.db images images.tsv \ --tsv \ -c id int \ -c score float",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/131/reactions"", ""total_count"": 1, ""+1"": 1, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 675839512,MDU6SXNzdWU2NzU4Mzk1MTI=,132,Features for enabling and disabling WAL mode,9599,closed,0,,,5,2020-08-10T03:25:44Z,2020-08-10T18:59:35Z,2020-08-10T18:59:35Z,OWNER,,I finally figured out how to enable WAL - turns out it's a property of the database file itself: https://github.com/simonw/til/blob/master/sqlite/enabling-wal-mode.md,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/132/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 677839979,MDU6SXNzdWU2Nzc4Mzk5Nzk=,133,Release a sdist to PyPI,9599,closed,0,,,1,2020-08-12T16:55:09Z,2020-08-12T17:05:06Z,2020-08-12T17:05:06Z,OWNER,,https://pypi.org/project/sqlite-utils/#files currently just has a wheel. I need this to package for homebrew: https://github.com/simonw/homebrew-datasette/issues/10,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/133/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 683804172,MDU6SXNzdWU2ODM4MDQxNzI=,134,--load-extension option for sqlite-utils query,9599,closed,0,,,4,2020-08-21T20:12:42Z,2020-08-21T21:06:26Z,2020-08-21T20:54:19Z,OWNER,,"I got this error: ``` % sqlite-utils calands.db 'create table superunits_with_maps_view_concrete as select * from superunits_with_maps_view' Traceback (most recent call last): ... cursor = db.conn.execute(sql, dict(param)) sqlite3.OperationalError: no such function: AsGeoJSON ``` A `--load-extension=/usr/local/lib/mod_spatialite.dylib` option (imitating the same option for Datasette) would help.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/134/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 683805434,MDU6SXNzdWU2ODM4MDU0MzQ=,135,Code for finding SpatiaLite in the usual locations,9599,closed,0,,,3,2020-08-21T20:15:34Z,2022-02-05T00:04:26Z,2020-08-21T20:30:13Z,OWNER,,"I built this for `shapefile-to-sqlite` but it would be useful in `sqlite-utils` too: https://github.com/simonw/shapefile-to-sqlite/blob/e754d0747ca2facf9a7433e2d5d15a6a37a9cf6e/shapefile_to_sqlite/utils.py#L16-L19 ```python SPATIALITE_PATHS = ( ""/usr/lib/x86_64-linux-gnu/mod_spatialite.so"", ""/usr/local/lib/mod_spatialite.dylib"", ) ``` https://github.com/simonw/shapefile-to-sqlite/blob/e754d0747ca2facf9a7433e2d5d15a6a37a9cf6e/shapefile_to_sqlite/utils.py#L105-L109 ```python def find_spatialite(): for path in SPATIALITE_PATHS: if os.path.exists(path): return path return None ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/135/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 683812642,MDU6SXNzdWU2ODM4MTI2NDI=,136,--load-extension=spatialite shortcut option,9599,closed,0,,,3,2020-08-21T20:31:25Z,2022-02-05T00:04:26Z,2020-10-16T19:14:32Z,OWNER,,In conjunction with #135 - this would do the same thing as `--load-extension=path-to-spatialite` (see #134),140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/136/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 683830416,MDU6SXNzdWU2ODM4MzA0MTY=,137,--load-extension for other sqlite-utils commands,9599,closed,0,,,1,2020-08-21T21:12:56Z,2020-10-16T19:14:32Z,2020-10-16T19:14:32Z,OWNER,,"e.g. for this: ``` calands-datasette % sqlite-utils tables calands.db --counts [{""table"": ""spatial_ref_sys"", ""count"": 4924}, {""table"": ""spatialite_history"", ""count"": 14}, {""table"": ""sqlite_sequence"", ""count"": 1}, {""table"": ""geometry_columns"", ""count"": 2}, {""table"": ""spatial_ref_sys_aux"", ""count"": 4873}, {""table"": ""views_geometry_columns"", ""count"": 0}, {""table"": ""virts_geometry_columns"", ""count"": 0}, {""table"": ""geometry_columns_statistics"", ""count"": 2}, {""table"": ""views_geometry_columns_statistics"", ""count"": 0}, {""table"": ""virts_geometry_columns_statistics"", ""count"": 0}, {""table"": ""geometry_columns_field_infos"", ""count"": 0}, {""table"": ""views_geometry_columns_field_infos"", ""count"": 0}, {""table"": ""virts_geometry_columns_field_infos"", ""count"": 0}, {""table"": ""geometry_columns_time"", ""count"": 2}, {""table"": ""geometry_columns_auth"", ""count"": 2}, {""table"": ""views_geometry_columns_auth"", ""count"": 0}, {""table"": ""virts_geometry_columns_auth"", ""count"": 0}, Traceback (most recent call last): File ""/usr/local/bin/sqlite-utils"", line 8, in sys.exit(cli()) File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/click/core.py"", line 829, in __call__ return self.main(*args, **kwargs) File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/click/core.py"", line 782, in main rv = self.invoke(ctx) File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/click/core.py"", line 1259, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/click/core.py"", line 1066, in invoke return ctx.invoke(self.callback, **ctx.params) File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/click/core.py"", line 610, in invoke return callback(*args, **kwargs) File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/sqlite_utils/cli.py"", line 143, in tables for line in output_rows(_iter(), headers, nl, arrays, json_cols): File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/sqlite_utils/cli.py"", line 922, in output_rows for row, next_row in itertools.zip_longest(current_iter, next_iter): File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/sqlite_utils/cli.py"", line 123, in _iter row.append(db[name].count) File ""/usr/local/Cellar/sqlite-utils/2.15.1/libexec/lib/python3.8/site-packages/sqlite_utils/db.py"", line 458, in count return self.db.conn.execute( sqlite3.OperationalError: no such module: VirtualSpatialIndex ``` The `tables` command could take `--load-extension` too - as could `rows` and other similar commands. Follow-on from #134 ",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/137/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 684118950,MDU6SXNzdWU2ODQxMTg5NTA=,138,extracts= doesn't configure foreign keys,9599,closed,0,,,2,2020-08-23T05:21:15Z,2020-09-24T22:47:01Z,2020-09-24T22:46:52Z,OWNER,,"In using `extracts=` for `shapefiles-to-sqlite` in https://github.com/simonw/shapefile-to-sqlite/issues/9 I've run into a couple of pretty serious flaws: - The columns in the original table are still `TEXT` even when the foreign key they are supposed to reference is an `INTEGER` - which means Datasette foreign key features don't actually work - Those foreign key relationships aren't setup automatically - creating them is left as an exercise for the developer",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/138/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 686978131,MDU6SXNzdWU2ODY5NzgxMzE=,139,"insert_all(..., alter=True) should work for new columns introduced after the first 100 records",96218,closed,0,,,7,2020-08-27T06:25:25Z,2020-08-28T22:48:51Z,2020-08-28T22:30:14Z,CONTRIBUTOR,,"Is there a way to make `.insert_all()` work properly when new columns are introduced outside the first 100 records (with or without the `alter=True` argument)? I'm using `.insert_all()` to bulk insert ~3-4k records at a time and it is common for records to need to introduce new columns. However, if new columns are introduced after the first 100 records, `sqlite_utils` doesn't even raise the `OperationalError: table ... has no column named ...` exception; it just silently drops the extra data and moves on. It took me a while to find this little snippet in the [documentation for `.insert_all()`](https://sqlite-utils.readthedocs.io/en/stable/python-api.html#bulk-inserts) (it's not mentioned under [Adding columns automatically on insert/update](https://sqlite-utils.readthedocs.io/en/stable/python-api.html#bulk-inserts)): > The column types used in the CREATE TABLE statement are automatically derived from the types of data in that first batch of rows. **_Any additional or missing columns in subsequent batches will be ignored._** I tried changing the `batch_size` argument to the total number of records, but it seems only to effect the number of rows that are committed at a time, and has no influence on this problem. Is there a way around this that you would suggest? It seems like it should raise an exception at least.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/139/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 413842611,MDU6SXNzdWU0MTM4NDI2MTE=,14,Utilities for adding indexes,9599,closed,0,,,3,2019-02-24T16:57:28Z,2019-02-24T19:11:28Z,2019-02-24T19:11:28Z,OWNER,,"Both in the Python API and the CLI tool. For the CLI tool this should work: $ sqlite-utils create-index mydb.db mytable col1 col2 This will create a compound index across col1 and col2. The name of the index will be automatically chosen unless you use the `--name=...` option. Support a `--unique` option too.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/14/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 688351054,MDU6SXNzdWU2ODgzNTEwNTQ=,140,Idea: insert-files mechanism for adding extra columns with fixed values,9599,open,0,,,1,2020-08-28T20:57:36Z,2022-03-20T19:45:45Z,,OWNER,,"Say for example you want to populate a `file_type` column with the value `gif`. That could work like this: ``` sqlite-utils insert-files gifs.db images *.gif \ -c path -c md5 -c last_modified:mtime \ -c file_type:text:gif --pk=path ``` So a column defined as a `text` column with a value that follows a second colon.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/140/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 688352145,MDU6SXNzdWU2ODgzNTIxNDU=,141,insert-files support for compressed values,9599,open,0,,,0,2020-08-28T20:59:46Z,2020-09-24T20:36:08Z,,OWNER,,"The `sqlar` format supports this, it would be useful if `insert-files` could support this too. https://www.sqlite.org/sqlar.html",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/141/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 688389933,MDU6SXNzdWU2ODgzODk5MzM=,143,Move to GitHub Actions CI,9599,closed,0,,,1,2020-08-28T22:34:11Z,2020-08-28T22:41:35Z,2020-08-28T22:41:35Z,OWNER,,,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/143/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 688395275,MDU6SXNzdWU2ODgzOTUyNzU=,144,Run some tests against numpy,9599,closed,0,,,2,2020-08-28T22:53:00Z,2020-08-28T22:57:05Z,2020-08-28T22:57:04Z,OWNER,,"Accidentally removed in #143: https://github.com/simonw/sqlite-utils/blob/d7d3f962861ef32c5ead8f514c8756f5b6f7c4a0/.travis.yml#L18-L19",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/144/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 688659182,MDU6SXNzdWU2ODg2NTkxODI=,145,Bug when first record contains fewer columns than subsequent records,96218,closed,0,,,2,2020-08-30T05:44:44Z,2020-09-08T23:21:23Z,2020-09-08T23:21:23Z,CONTRIBUTOR,,"`insert_all()` selects the maximum batch size based on the number of fields in the first record. If the first record has fewer fields than subsequent records (and `alter=True` is passed), this can result in SQL statements with more than the maximum permitted number of host parameters. This situation is perhaps unlikely to occur, but could happen if the first record had, say, 10 columns, such that `batch_size` (based on `SQLITE_MAX_VARIABLE_NUMBER = 999`) would be 99. If the next 98 rows had 11 columns, the resulting SQL statement for the first batch would have `10 * 1 + 11 * 98 = 1088` host parameters (and subsequent batches, if the data were consistent from thereon out, would have `99 * 11 = 1089`). I suspect that this bug is masked somewhat by the fact that while: > [`SQLITE_MAX_VARIABLE_NUMBER`](https://www.sqlite.org/limits.html#max_variable_number) ... defaults to 999 for SQLite versions prior to 3.32.0 (2020-05-22) or 32766 for SQLite versions after 3.32.0. it is common that it is increased at compile time. Debian-based systems, for example, seem to ship with a version of sqlite compiled with `SQLITE_MAX_VARIABLE_NUMBER` set to 250,000, and I believe this is the case for homebrew installations too. A test for this issue might look like this: ```python def test_columns_not_in_first_record_should_not_cause_batch_to_be_too_large(fresh_db): # sqlite on homebrew and Debian/Ubuntu etc. is typically compiled with # SQLITE_MAX_VARIABLE_NUMBER set to 250,000, so we need to exceed this value to # trigger the error on these systems. THRESHOLD = 250000 extra_columns = 1 + (THRESHOLD - 1) // 99 records = [ {""c0"": ""first record""}, # one column in first record -> batch_size = 100 # fill out the batch with 99 records with enough columns to exceed THRESHOLD *[ dict([(""c{}"".format(i), j) for i in range(extra_columns)]) for j in range(99) ] ] try: fresh_db[""too_many_columns""].insert_all(records, alter=True) except sqlite3.OperationalError: raise ``` The best solution, I think, is simply to process all the records when determining columns, column types, and the batch size. In my tests this doesn't seem to be particularly costly at all, and cuts out a lot of complications (including obviating my implementation of #139 at #142). I'll raise a PR for your consideration. ",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/145/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 688670158,MDU6SXNzdWU2ODg2NzAxNTg=,147,SQLITE_MAX_VARS maybe hard-coded too low,96218,open,0,,,7,2020-08-30T07:26:45Z,2021-02-15T21:27:55Z,,CONTRIBUTOR,,"I came across this while about to open an issue and PR against the documentation for `batch_size`, which is a bit incomplete. As mentioned in #145, while: > [`SQLITE_MAX_VARIABLE_NUMBER`](https://www.sqlite.org/limits.html#max_variable_number) ... defaults to 999 for SQLite versions prior to 3.32.0 (2020-05-22) or 32766 for SQLite versions after 3.32.0. it is common that it is increased at compile time. Debian-based systems, for example, seem to ship with a version of sqlite compiled with SQLITE_MAX_VARIABLE_NUMBER set to 250,000, and I believe this is the case for homebrew installations too. In working to understand what `batch_size` was actually doing and why, I realized that by setting `SQLITE_MAX_VARS` in `db.py` to match the value my sqlite was compiled with (I'm on Debian), I was able to decrease the time to `insert_all()` my test data set (~128k records across 7 tables) from ~26.5s to ~3.5s. Given that this about .05% of my total dataset, this is time I am keen to save... Unfortunately, it seems that `sqlite3` in the python standard library doesn't expose the `get_limit()` C API (even though `pysqlite` used to), so it's hard to know what value sqlite has been compiled with (note that this could mean, I suppose, that it's less than 999, and even hardcoding `SQLITE_MAX_VARS` to the conservative default might not be adequate. It can also be lowered -- but not raised -- at runtime). The best I could come up with is `echo """" | sqlite3 -cmd "".limits variable_number""` (only available in `sqlite >= 2015-05-07 (3.8.10)`). Obviously this couldn't be relied upon in `sqlite_utils`, but I wonder what your opinion would be about exposing `SQLITE_MAX_VARS` as a user-configurable parameter (with suitable ""here be dragons"" warnings)? I'm going to go ahead and monkey-patch it for my purposes in any event, but it seems like it might be worth considering.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/147/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 695276328,MDU6SXNzdWU2OTUyNzYzMjg=,148,More attractive indentation of created FTS table schema,9599,closed,0,,,1,2020-09-07T16:49:30Z,2020-09-07T18:12:50Z,2020-09-07T18:12:50Z,OWNER,,"On https://github-to-sqlite.dogsheep.net/github/licenses_fts the create table SQL is displayed as: ```sql CREATE VIRTUAL TABLE [licenses_fts] USING FTS5 ( [name], content=[licenses] ); ``` It would be more aesthetically pleasing if it looked like this: ```sql CREATE VIRTUAL TABLE [licenses_fts] USING FTS5 ( [name], content=[licenses] ); ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/148/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 695319258,MDU6SXNzdWU2OTUzMTkyNTg=,149,"FTS table with 7 rows has _fts_docsize table with 9,141 rows",9599,closed,0,,,10,2020-09-07T18:06:16Z,2020-09-07T21:16:34Z,2020-09-07T21:16:34Z,OWNER,,"I'm seeing a weird issue with some of the SQLite databases that I am using with the FTS5 module. I have a database with a `licenses` table that contains 7 rows: The FTS table also has 7 rows: Somehow the accompanying `licenses_fts_docsize` shadow table now has 9,141 rows in it! And `licenses_fts_data` has 41 rows - should I expect that to have 7 rows? I have a hunch that it might be a problem with the triggers. These are the triggers that are updating that FTS table: | type | name | tbl_name | rootpage | sql | | --- | --- | --- | --- | --- | | trigger | licenses_ai | licenses | 0 | `CREATE TRIGGER [licenses_ai] AFTER INSERT ON [licenses] BEGIN INSERT INTO [licenses_fts] (rowid, [name]) VALUES (new.rowid, new.[name]); END` | | trigger | licenses_ad | licenses | 0 | `CREATE TRIGGER [licenses_ad] AFTER DELETE ON [licenses] BEGIN INSERT INTO [licenses_fts] ([licenses_fts], rowid, [name]) VALUES('delete', old.rowid, old.[name]); END` | | trigger | licenses_au | licenses | 0 | `CREATE TRIGGER [licenses_au] AFTER UPDATE ON [licenses] BEGIN INSERT INTO [licenses_fts] ([licenses_fts], rowid, [name]) VALUES('delete', old.rowid, old.[name]); INSERT INTO [licenses_fts] (rowid, [name]) VALUES (new.rowid, new.[name]); END` |",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/149/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 413857257,MDU6SXNzdWU0MTM4NTcyNTc=,15,Ability to add columns to tables,9599,closed,0,,,0,2019-02-24T19:20:51Z,2019-02-24T20:04:40Z,2019-02-24T20:04:40Z,OWNER,,"Makes sense to do this before foreign keys in #2 Python: db[""table""].add_column(""new_column"", int) CLI: $ sqlite-utils add-column table new_column INTEGER ",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/15/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 695359607,MDU6SXNzdWU2OTUzNTk2MDc=,150,Feature for tracing SQL queries,9599,closed,0,,,0,2020-09-07T19:43:08Z,2020-09-07T21:57:01Z,2020-09-07T21:57:01Z,OWNER,,"Debugging `sqlite-utils` when something weird happens (e.g. #149) can be a bit tricky since it runs a bunch of different SQL statements behind the scenes. An optional ""tracing"" mechanism for seeing what SQL is being executed would be useful.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/150/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 695376054,MDU6SXNzdWU2OTUzNzYwNTQ=,152,Turn on recursive_triggers by default,9599,closed,0,,,2,2020-09-07T20:26:36Z,2020-09-07T21:17:48Z,2020-09-07T20:45:14Z,OWNER,,"https://www.sqlite.org/pragma.html#pragma_recursive_triggers says: > Prior to SQLite [version 3.6.18](https://www.sqlite.org/releaselog/3_6_18.html) (2009-09-11), recursive triggers were not supported. The behavior of SQLite was always as if this pragma was set to OFF. Support for recursive triggers was added in version 3.6.18 but was initially turned OFF by default, for compatibility. Recursive triggers may be turned on by default in future versions of SQLite. So I think the fix for the complex issue in #149 is to turn on `recursive_triggers` globally by default for `sqlite-utils`. _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/issues/149#issuecomment-688499924_",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/152/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 695377804,MDU6SXNzdWU2OTUzNzc4MDQ=,153,table.optimize() should delete junk rows from *_fts_docsize,9599,closed,0,,,3,2020-09-07T20:31:09Z,2020-09-24T20:35:46Z,2020-09-07T21:16:33Z,OWNER,,"> The second challenge here is cleaning up all of those junk rows in existing `*_fts_docsize` tables. Doing that just to the demo database from https://github-to-sqlite.dogsheep.net/github.db dropped its size from 22MB to 16MB! Here's the SQL: > ```sql > DELETE FROM [licenses_fts_docsize] WHERE id NOT IN ( > SELECT rowid FROM [licenses_fts]); > ``` > I can do that as part of the existing `table.optimize()` method, which optimizes FTS tables. _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/issues/149#issuecomment-688501064_",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/153/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 695441530,MDU6SXNzdWU2OTU0NDE1MzA=,154,OperationalError: cannot change into wal mode from within a transaction,9599,open,0,,,2,2020-09-07T23:42:44Z,2020-09-07T23:47:10Z,,OWNER,,"I'm getting this error when running: sqlite-utils enable-wal beta.db `OperationalError: cannot change into wal mode from within a transaction` I'm worried that maybe that's because of this new code from #152: https://github.com/simonw/sqlite-utils/blob/deb2eb013ff85bbc828ebc244a9654f0d9c3139e/sqlite_utils/db.py#L128-L129",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/154/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 696045581,MDU6SXNzdWU2OTYwNDU1ODE=,155,rebuild-fts command and table.rebuild_fts() method,9599,closed,0,,,2,2020-09-08T17:19:26Z,2020-09-24T20:35:46Z,2020-09-08T23:16:10Z,OWNER,,"https://sqlite.org/forum/forumpost/fa777fff86 > Easiest thing would be to run a 'rebuild' to rebuild the FTS index from scratch based on the contents of the content table. i.e. > > INSERT INTO licenses_fts(licenses_fts) VALUES('rebuild'); > > https://www.sqlite.org/fts5.html#the_rebuild_command",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/155/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 697179806,MDU6SXNzdWU2OTcxNzk4MDY=,157,sqlite-utils add-foreign-keys command,9599,closed,0,,5896742,2,2020-09-09T21:44:30Z,2020-09-24T20:34:50Z,2020-09-20T20:14:30Z,OWNER,,"Like `add-foreign-key` but can do multiple foreign keys at once. Inspired by https://github.com/simonw/calands-datasette/blob/99de39dd80a906f5c1f16724467b0cd55ba4ef36/build.sh which does this: ``` sqlite-utils add-foreign-key calands.db units_with_maps ACCESS_TYP sqlite-utils add-foreign-key calands.db units_with_maps AGNCY_NAME sqlite-utils add-foreign-key calands.db units_with_maps AGNCY_LEV sqlite-utils add-foreign-key calands.db units_with_maps AGNCY_TYP sqlite-utils add-foreign-key calands.db units_with_maps LAYER sqlite-utils add-foreign-key calands.db units_with_maps MNG_AGENCY sqlite-utils add-foreign-key calands.db units_with_maps MNG_AG_LEV sqlite-utils add-foreign-key calands.db units_with_maps MNG_AG_TYP sqlite-utils add-foreign-key calands.db units_with_maps COUNTY sqlite-utils add-foreign-key calands.db units_with_maps DES_TP ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/157/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 702386948,MDU6SXNzdWU3MDIzODY5NDg=,159,.delete_where() does not auto-commit (unlike .insert() or .upsert()),11712349,open,0,,,9,2020-09-16T01:55:52Z,2023-04-01T17:21:05Z,,NONE,,"When you use the delete_where() function on a table, it never commits.... Is that intentional?",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/159/reactions"", ""total_count"": 1, ""+1"": 1, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 413867537,MDU6SXNzdWU0MTM4Njc1Mzc=,16,add_column() should support REFERENCES {other_table}({other_column}),9599,closed,0,,,4,2019-02-24T21:00:45Z,2019-05-29T05:17:59Z,2019-05-29T04:56:18Z,OWNER,,Related to #2 ,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/16/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 705190723,MDU6SXNzdWU3MDUxOTA3MjM=,160,"table.enable_fts(..., replace=True)",9599,closed,0,,5896742,1,2020-09-20T21:36:23Z,2020-09-24T20:35:47Z,2020-09-20T22:05:51Z,OWNER,,"I noticed that https://til.simonwillison.net/ search doesn't use porter stemming. I'd like to add that, but since [the build script](https://github.com/simonw/til/blob/9d3f0fca30e94df3970df52b0447907a077e4673/build_database.py) always operates on an existing database (to avoid re-rendering markdown and re-building image thumbnails) I'd like it to only add porter stemming if it's not there already. So I'd like to be able to say ""set up FTS to look like this, and fix it if it doesn't"". I think the neatest way to do that is with a `replace=True` argument to `.enable_fts()`, for consistency with `def .create_view(self, name, sql, replace=True)`. So the `replace=True` argument would check and see if the configured FTS exists already with the correct options (columns, stemming, triggers) - and if any of those are incorrect it would call `.disable_fts()` and then create a new FTS configuration with the correct options. ",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/160/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 705995722,MDU6SXNzdWU3MDU5OTU3MjI=,162,A decorator for registering custom SQL functions,9599,closed,0,,,2,2020-09-22T00:18:32Z,2020-09-22T00:40:44Z,2020-09-22T00:32:17Z,OWNER,,"Syntactic sugar for `db.conn.create_function` - it would work something like this: ```python db = sqlite_utils.Database(""mydb.db"") @db.register_function def scramble(text): chars = list(text) random.shuffle(chars) return """".join(chars) ``` The decorator would inspect the function to find its name and arity (number of arguments). Having run the above you could then do: ```python db.execute(""select scramble('hello')"").fetchall() ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/162/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 706001517,MDU6SXNzdWU3MDYwMDE1MTc=,163,Idea: conversions= could take Python functions,9599,open,0,,,4,2020-09-22T00:37:12Z,2021-12-20T00:56:52Z,,OWNER,,"Right now you use `conversions=` like this: ```python db[""example""].insert({ ""name"": ""The Bigfoot Discovery Museum"" }, conversions={""name"": ""upper(?)""}) ``` How about if you could optionally provide a Python function (or a lambda) like this? ```python db[""example""].insert({ ""name"": ""The Bigfoot Discovery Museum"" }, conversions={""name"": lambda s: s.upper()}) ``` This would work by creating a random name for that function, registering it (similar to #162), executing the SQL and then un-registering the custom function at the end.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/163/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 706017416,MDU6SXNzdWU3MDYwMTc0MTY=,164,sqlite-utils transform sub-command,9599,closed,0,,5897911,4,2020-09-22T01:32:20Z,2020-09-24T20:34:50Z,2020-09-22T07:48:05Z,OWNER,,The `.transform()` method in #114 warrants an equivalent CLI tool.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/164/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 706091046,MDU6SXNzdWU3MDYwOTEwNDY=,165,Make .transform() a keyword arguments only function,9599,closed,0,,5897911,0,2020-09-22T05:37:29Z,2020-09-24T20:35:47Z,2020-09-22T06:39:12Z,OWNER,,And rename the first argument from `columns=` to `types=`,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/165/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 706098005,MDU6SXNzdWU3MDYwOTgwMDU=,167,Review the foreign key pragma stuff,9599,closed,0,,5897911,1,2020-09-22T05:55:20Z,2020-09-23T00:13:02Z,2020-09-23T00:13:02Z,OWNER,,"> It is not possible to enable or disable foreign key constraints in the middle of a multi-statement transaction (when SQLite is not in autocommit mode). Attempting to do so does not return an error; it simply has no effect. https://sqlite.org/foreignkeys.html",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/167/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 706167456,MDU6SXNzdWU3MDYxNjc0NTY=,168,Automate (as much as possible) updates published to Homebrew,9599,closed,0,,,2,2020-09-22T08:08:37Z,2020-11-09T07:43:30Z,2020-11-09T07:43:30Z,OWNER,,I'd like to get new `sqlite-utils` (and Datasette) releases submitted to Homebrew as painlessly as possible.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/168/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 706757891,MDU6SXNzdWU3MDY3NTc4OTE=,169,"Progress bar for ""sqlite-utils extract""",9599,closed,0,,5897911,0,2020-09-22T23:40:21Z,2020-09-24T20:34:50Z,2020-09-23T00:02:40Z,OWNER,,"> Since these operations could take a long time against large tables, it would be neat if there was a progress bar option for the CLI command. > > The operations are full table scans so calculating progress shouldn't be too difficult. _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/issues/42#issuecomment-513246831_",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/169/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 413868452,MDU6SXNzdWU0MTM4Njg0NTI=,17,Improve and document foreign_keys=... argument to insert/create/etc,9599,closed,0,,,7,2019-02-24T21:09:11Z,2019-02-24T23:45:48Z,2019-02-24T23:45:48Z,OWNER,,"The `foreign_keys=` argument to `table.insert_all()` and friends can be used to specify foreign key relationships that should be created. It is not yet documented. It also requires you to specify the SQLite type of each column, even though this can be detected by introspecting the referenced table: cols = [c for c in self.db[other_table].columns if c.name == other_column] cols[0].type Relates to #2 ",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/17/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 706768798,MDU6SXNzdWU3MDY3Njg3OTg=,170,Release notes for 2.20,9599,closed,0,,5897911,1,2020-09-23T00:13:22Z,2020-09-23T00:31:25Z,2020-09-23T00:31:25Z,OWNER,,https://github.com/simonw/sqlite-utils/compare/2.19...b8e004,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/170/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 707407567,MDU6SXNzdWU3MDc0MDc1Njc=,171,Idea: transitive closure tables for tree structures,649467,closed,0,,,2,2020-09-23T14:17:33Z,2020-10-22T04:38:35Z,2020-10-22T04:07:14Z,NONE,,"I just read that sqlite has a transitive closure table extension using a virtual table in order to represent trees: https://charlesleifer.com/blog/querying-tree-structures-in-sqlite-using-python-and-the-transitive-closure-extension/ Even without this extension, though, a util to build a transitive closure table would allow trees to be queried easily. Since it relies on self-referential foreign keys, the relationships might even be able to be automatically detected. ",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/171/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 707427200,MDU6SXNzdWU3MDc0MjcyMDA=,172,Improve performance of extract operations,9599,closed,0,,,9,2020-09-23T14:40:50Z,2020-09-24T15:43:57Z,2020-09-24T15:43:57Z,OWNER,,"This command took about 12 minutes (against a 150MB file with 680,000 rows in it): ``` sqlite-utils extract salaries.db salaries \ 'Organization Group Code' 'Organization Group' \ --table 'organization_groups' \ --fk-column 'organization_group_id' \ --rename 'Organization Group Code' code \ --rename 'Organization Group' name ``` I'm pretty confident we can do better than that.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/172/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 707478649,MDU6SXNzdWU3MDc0Nzg2NDk=,173,Progress bar for sqlite-utils insert,9599,closed,0,,,6,2020-09-23T15:43:56Z,2021-11-01T08:42:24Z,2020-10-27T18:16:04Z,OWNER,,"It would be nice if `sqlite-utils insert` had a progress bar, for when it's churning through huge CSV files.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/173/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 708261775,MDU6SXNzdWU3MDgyNjE3NzU=,175,Add docs for .transform(column_order=),9599,closed,0,,,3,2020-09-24T15:19:04Z,2020-09-24T20:35:48Z,2020-09-24T16:00:56Z,OWNER,,"> Need to update docs for `.transform()` now that `column_order=` is available. _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/pull/174#discussion_r494403327_ Maybe also add this as an option to `sqlite-utils transform` - since reordering columns is actually a pretty nice capability.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/175/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 708293114,MDU6SXNzdWU3MDgyOTMxMTQ=,176,sqlite-utils transform column order option,9599,closed,0,,,2,2020-09-24T16:01:21Z,2020-09-24T20:34:51Z,2020-09-24T16:11:59Z,OWNER,,Split from #175,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/176/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 708301810,MDU6SXNzdWU3MDgzMDE4MTA=,177,Simplify .transform(drop_foreign_keys=) and sqlite-transform --drop-foreign-key,9599,closed,0,,,1,2020-09-24T16:13:50Z,2020-09-24T20:35:03Z,2020-09-24T16:19:13Z,OWNER,,"These both currently require you to provide three strings, for `column`, `other_table`, `other_column`. Just providing `column` should be enough information.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/177/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 709577625,MDU6SXNzdWU3MDk1Nzc2MjU=,179,sqlite-utils transform/insert --detect-types,9599,closed,0,,,4,2020-09-26T17:28:55Z,2021-06-19T03:36:16Z,2021-06-19T03:36:05Z,OWNER,,"Idea from https://github.com/simonw/datasette-edit-tables/issues/13 - provide Python utility methods and accompanying CLI options for detecting the likely types of TEXT columns. So if you have a text column that actually contained exclusively integer string values, it can let you know and let you run transform against it.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/179/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 413871266,MDU6SXNzdWU0MTM4NzEyNjY=,18,.insert/.upsert/.insert_all/.upsert_all should add missing columns,9599,closed,0,,4348046,2,2019-02-24T21:36:11Z,2019-05-25T00:42:11Z,2019-05-25T00:42:11Z,OWNER,,"This is a larger change, but it would be incredibly useful: if you attempt to insert or update a document with a field that does not currently exist in the underlying table, sqlite-utils should add the appropriate column for you.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/18/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 709861194,MDU6SXNzdWU3MDk4NjExOTQ=,180,Try running some tests using Hypothesis,9599,closed,0,,,1,2020-09-28T01:11:30Z,2020-10-19T04:51:55Z,2020-10-19T04:51:55Z,OWNER,,Inspired by this Twitter conversation: https://twitter.com/simonw/status/1310386009465479168,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/180/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 709920027,MDU6SXNzdWU3MDk5MjAwMjc=,181,"pk=[""id""] should have same effect as pk=""id""",9599,closed,0,,,1,2020-09-28T04:28:07Z,2020-10-14T21:59:47Z,2020-10-14T21:59:47Z,OWNER,,"``` In [11]: db['one'].insert({""id"": 1, ""name"": ""oentuh""}, pk=""id"") Out[11]:
In [12]: db['two'].insert({""id"": 1, ""name"": ""oentuh""}, pk=[""id""]) Out[12]:
In [13]: db['one'].schema Out[13]: 'CREATE TABLE [one] (\n [id] INTEGER PRIMARY KEY,\n [name] TEXT\n)' In [14]: db['two'].schema Out[14]: 'CREATE TABLE [two] (\n [id] INTEGER,\n [name] TEXT\n)' ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/181/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 711649325,MDU6SXNzdWU3MTE2NDkzMjU=,182,"Better handling of encodings other than utf-8 for ""sqlite-utils insert""",765871,closed,0,,,5,2020-09-30T05:43:48Z,2020-10-16T17:20:41Z,2020-10-16T17:18:52Z,NONE,,"Makefile: ``` data.db: curl -O http://maps.natalian.org/data.txt go run csv-write.go > data.csv sqlite-utils insert data.db travels data.csv --csv clean: rm data* ``` [csv-write.go](https://gist.github.com/kaihendry/dff2442de20d73f900026d13bf7a11d9) Error message is: ``` sqlite-utils insert data.db travels data.csv --csv Traceback (most recent call last): File ""/home/hendry/.local/bin/sqlite-utils"", line 8, in sys.exit(cli()) File ""/home/hendry/.local/lib/python3.8/site-packages/click/core.py"", line 829, in __call__ return self.main(*args, **kwargs) File ""/home/hendry/.local/lib/python3.8/site-packages/click/core.py"", line 782, in main rv = self.invoke(ctx) File ""/home/hendry/.local/lib/python3.8/site-packages/click/core.py"", line 1259, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File ""/home/hendry/.local/lib/python3.8/site-packages/click/core.py"", line 1066, in invoke return ctx.invoke(self.callback, **ctx.params) File ""/home/hendry/.local/lib/python3.8/site-packages/click/core.py"", line 610, in invoke return callback(*args, **kwargs) File ""/home/hendry/.local/lib/python3.8/site-packages/sqlite_utils/cli.py"", line 614, in insert insert_upsert_implementation( File ""/home/hendry/.local/lib/python3.8/site-packages/sqlite_utils/cli.py"", line 553, in insert_upsert_implementation headers = next(reader) File ""/usr/lib/python3.8/codecs.py"", line 322, in decode (result, consumed) = self._buffer_decode(data, self.errors, final) UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe3 in position 1234: invalid continuation byte make: *** [Makefile:4: data.db] Error 1 [hendry@t14s datasette-map]$ sqlite-utils --version sqlite-utils, version 2.19 ``` Little bit surprised if Go is spewing out bad Unicode, but I'm not sure how to grok `position 1234`.. ",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/182/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 712316959,MDU6SXNzdWU3MTIzMTY5NTk=,183,Try out GitHub code scanning,9599,closed,0,,,1,2020-09-30T22:16:14Z,2020-09-30T22:23:44Z,2020-09-30T22:23:44Z,OWNER,,https://github.blog/2020-09-30-code-scanning-is-now-available/,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/183/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 718952107,MDU6SXNzdWU3MTg5NTIxMDc=,185,Use db[table] consistently in documentation,9599,closed,0,,,0,2020-10-11T23:39:12Z,2020-10-12T00:13:41Z,2020-10-12T00:13:41Z,OWNER,,"The Python docs have a bunch of examples like this: https://sqlite-utils.readthedocs.io/en/stable/python-api.html ```python dogs.enable_fts([""name"", ""twitter""], create_triggers=True) ``` This would be easier for people to understand if it looked like this instead: ```python db[""dogs""].enable_fts([""name"", ""twitter""], create_triggers=True) ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/185/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 722816436,MDU6SXNzdWU3MjI4MTY0MzY=,186,.extract() shouldn't extract null values,9599,open,0,,,7,2020-10-16T02:41:08Z,2021-08-12T12:32:14Z,,OWNER,,"This almost works, but it creates a rogue `type` record with a value of None. ``` In [1]: import sqlite_utils In [2]: db = sqlite_utils.Database(memory=True) In [5]: db[""creatures""].insert_all([ {""id"": 1, ""name"": ""Simon"", ""type"": None}, {""id"": 2, ""name"": ""Natalie"", ""type"": None}, {""id"": 3, ""name"": ""Cleo"", ""type"": ""dog""}], pk=""id"") Out[5]:
In [7]: db[""creatures""].extract(""type"") Out[7]:
In [8]: list(db[""creatures""].rows) Out[8]: [{'id': 1, 'name': 'Simon', 'type_id': None}, {'id': 2, 'name': 'Natalie', 'type_id': None}, {'id': 3, 'name': 'Cleo', 'type_id': 2}] In [9]: db[""type""] Out[9]:
In [10]: list(db[""type""].rows) Out[10]: [{'id': 1, 'type': None}, {'id': 2, 'type': 'dog'}] ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/186/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",, 723460107,MDU6SXNzdWU3MjM0NjAxMDc=,187,Maybe: Utility method / CLI tool for initializing SpatiaLite,9599,closed,0,,,2,2020-10-16T19:04:03Z,2022-02-05T00:04:26Z,2020-10-16T19:15:13Z,OWNER,,"> I think this should initialize SpatiaLite against the current database if it has not been initialized already. > > Relevant code: https://github.com/simonw/shapefile-to-sqlite/blob/e754d0747ca2facf9a7433e2d5d15a6a37a9cf6e/shapefile_to_sqlite/utils.py#L112-L126",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/187/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 723708310,MDU6SXNzdWU3MjM3MDgzMTA=,188,About loading spatialite,30607,closed,0,,,1,2020-10-17T08:47:02Z,2022-02-05T00:04:26Z,2020-10-17T08:52:58Z,NONE,,"Hi @simonw , If I run ``` sqlite3 .load /usr/local/lib/mod_spatialite.so select spatialite_version(); ``` I have `5.0.0`. ![image](https://user-images.githubusercontent.com/30607/96332706-d8cd3300-1065-11eb-906b-daf99963198e.png) If I run ``` sqlite-utils :memory: ""select spatialite_version()"" --load-extension=spatialite ``` I have ``` Traceback (most recent call last): File ""/home/aborruso/.local/bin/sqlite-utils"", line 8, in sys.exit(cli()) File ""/home/aborruso/.local/lib/python3.8/site-packages/click/core.py"", line 829, in __call__ return self.main(*args, **kwargs) File ""/home/aborruso/.local/lib/python3.8/site-packages/click/core.py"", line 782, in main rv = self.invoke(ctx) File ""/home/aborruso/.local/lib/python3.8/site-packages/click/core.py"", line 1259, in invoke return _process_result(sub_ctx.command.invoke(sub_ctx)) File ""/home/aborruso/.local/lib/python3.8/site-packages/click/core.py"", line 1066, in invoke return ctx.invoke(self.callback, **ctx.params) File ""/home/aborruso/.local/lib/python3.8/site-packages/click/core.py"", line 610, in invoke return callback(*args, **kwargs) File ""/home/aborruso/.local/lib/python3.8/site-packages/sqlite_utils/cli.py"", line 936, in query _load_extensions(db, load_extension) File ""/home/aborruso/.local/lib/python3.8/site-packages/sqlite_utils/cli.py"", line 1326, in _load_extensions db.conn.load_extension(ext) TypeError: argument 1 must be str, not None ``` How to load properly spatialite extension in sqlite-utils? Thank you very muc",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/188/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 432217625,MDU6SXNzdWU0MzIyMTc2MjU=,19,Incorrect help text for enable-fts command,9599,closed,0,,4348046,0,2019-04-11T19:46:44Z,2019-05-25T00:44:31Z,2019-05-25T00:44:31Z,OWNER,,"I clearly copied-and-pasted this from the `tables` command without updating it: https://github.com/simonw/sqlite-utils/blob/0b1af42ead3b3902347951180b3364ce1942da6e/sqlite_utils/cli.py#L216-L222",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/19/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 731740458,MDU6SXNzdWU3MzE3NDA0NTg=,191,Idea: @db.register_function(deterministic=True),9599,closed,0,,,2,2020-10-28T19:45:18Z,2020-10-28T21:31:06Z,2020-10-28T21:31:06Z,OWNER,,"Python 3.8 added a `deterministic` parameter to `db.create_function()`: https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.create_function `sqlite-utils` could expose this in the decorator, only actually applying it if the Python version supports it (using feature detection) - since nothing will break if it's not applied. https://sqlite-utils.readthedocs.io/en/stable/python-api.html#registering-custom-sql-functions ```python db = Database(memory=True) @db.register_function(deterministic=True) def reverse_string(s): return """".join(reversed(list(s))) ```",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/191/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 735532751,MDU6SXNzdWU3MzU1MzI3NTE=,192,sqlite-utils search command,9599,closed,0,,6079500,9,2020-11-03T18:07:59Z,2020-11-08T17:07:01Z,2020-11-08T17:07:01Z,OWNER,,A command that knows how to run a search against a FTS enabled table and return results ranked by relevance.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/192/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 735648209,MDU6SXNzdWU3MzU2NDgyMDk=,193,--tsv output format option,9599,closed,0,,6079500,0,2020-11-03T21:31:18Z,2020-11-07T00:09:52Z,2020-11-07T00:09:52Z,OWNER,,"We already support `--csv` for output, and the `insert` command accepts `--tsv`. The output format options should accept `--tsv` too.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/193/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 735650864,MDU6SXNzdWU3MzU2NTA4NjQ=,194,3.0 release with some minor breaking changes,9599,closed,0,,6079500,3,2020-11-03T21:36:31Z,2020-11-08T17:19:35Z,2020-11-08T17:19:34Z,OWNER,,"While working on search (#192) I've spotted a few small changes I would like to make that would break backwards compatibility in minor ways, hence requiring a 3.x release. `db[table].search()` - I would like this to default to sorting by rank Also I'd like to free up the `-c` and `-f` options for other purposes from the standard output formats here: https://github.com/simonw/sqlite-utils/blob/43eae8b193d362f2b292df73e087ed6f10838144/sqlite_utils/cli.py#L48-L58 I'd like `-f` to be used to indicate a full-text search column during an insert and `-c` to indicate a column (so you can specify which columns you want to output).",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/194/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 736520310,MDU6SXNzdWU3MzY1MjAzMTA=,196,Introspect if table is FTS4 or FTS5,9599,closed,0,,,19,2020-11-05T00:45:50Z,2020-11-05T03:54:07Z,2020-11-05T03:54:07Z,OWNER,,"> I want `.search()` to work against both FTS5 and FTS4 tables - but sort by rank should only work for FTS5. > > This means I need to be able to introspect and tell if a table is FTS4 or FTS5. _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/issues/192#issuecomment-722054264_",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/196/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 737153927,MDU6SXNzdWU3MzcxNTM5Mjc=,197,Rethink how table.search() method works,9599,closed,0,,6079500,5,2020-11-05T18:04:34Z,2020-11-08T17:07:37Z,2020-11-08T17:07:37Z,OWNER,,"I need to improve this method to help build `sqlite-utils search` in #192 (PR is #195). The challenge is deciding how it should handle sorting by relevance - especially since that is easy in FTS5 but not at all easy in FTS4. > Latest test failure: > ``` > 114 -> assert [(""racoons are biting trash pandas"", ""USA"", ""bar"")] == table.search( > 115 ""bite"", order=""rowid"" > 116 ) > 117 > 118 > 119 def test_optimize_fts(fresh_db): > (Pdb) table.search(""bite"") > [(2, 'racoons are biting trash pandas', 'USA', 'bar', -9.641434262948206e-07)] > ``` > The problem here is that the `table.search()` method now behaves differently for FTS4 v.s. FTS5 tables. > > With FTS4 you get back just the table columns. > > With FTS5 you also get back the `rowid` as the first column and the `rank` score as the last column. > > This is weird. It also makes me question whether having `.search()` return a list of tuples is the right API design. _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/pull/195#issuecomment-722542895_",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/197/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 737476423,MDU6SXNzdWU3Mzc0NzY0MjM=,198,Support order by relevance against FTS4,9599,closed,0,,,6,2020-11-06T05:36:31Z,2020-11-06T18:30:44Z,2020-11-06T18:30:44Z,OWNER,,"For #192 and #197 I've decided I want to be able to order by relevance in FTS4 as well as FTS5. This means I need to port over my work on bm25() from https://github.com/simonw/sqlite-fts4 (since I don't want to add a full dependency).",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/198/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 737855731,MDU6SXNzdWU3Mzc4NTU3MzE=,199,"@db.register_function(..., replace=False) to avoid double-registering custom functions",9599,closed,0,,,1,2020-11-06T15:39:21Z,2020-11-06T18:30:44Z,2020-11-06T18:30:44Z,OWNER,,"I'd like a mechanism to optionally avoid registering a custom function if it has already been registered. SQLite doesn't seem to offer a way to introspect registered custom functions so I'll need to track what has already been registered in `sqlite-utils` instead. > Should I register the custom `rank_bm25` SQLite function for every connection, or should I register it against the connection just the first time the user attempts an FTS4 search? I think I'd rather register it only if it is needed. _Originally posted by @simonw in https://github.com/simonw/sqlite-utils/issues/198#issuecomment-723145383_",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/199/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 349850687,MDU6SXNzdWUzNDk4NTA2ODc=,2,Mechanism for adding foreign keys to an existing table,9599,closed,0,,,1,2018-08-12T22:50:56Z,2019-02-24T21:34:41Z,2019-02-24T21:34:41Z,OWNER,,"SQLite does not have ALTER TABLE support for adding new foreign keys... but it turns out it's possible to make these changes without having to duplicate the entire table by carefully running `UPDATE sqlite_master SET sql=... WHERE type='table' AND name='X';` Here's how Django does it: https://github.com/django/django/blob/d3449faaa915a08c275b35de01e66a7ef6bdb2dc/django/db/backends/sqlite3/schema.py#L103-L125 And here's the official documentation about this: https://sqlite.org/lang_altertable.html#otheralter (scroll to the very bottom of the page)",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/2/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 432727685,MDU6SXNzdWU0MzI3Mjc2ODU=,20,JSON column values get extraneously quoted ,649467,closed,0,,4348046,1,2019-04-12T20:15:30Z,2019-05-25T00:57:19Z,2019-05-25T00:57:19Z,NONE,,"If the input to `sqlite-utils insert` includes a column that is a JSON array or object, `sqlite-utils query` will introduce an extra level of quoting on output: ``` # echo '[{""key"": [""one"", ""two"", ""three""]}]' | sqlite-utils insert t.db t - # sqlite-utils t.db 'select * from t' [{""key"": ""[\""one\"", \""two\"", \""three\""]""}] # sqlite3 t.db 'select * from t' [""one"", ""two"", ""three""] ``` This might require an imperfect solution, since sqlite3 doesn't have a JSON type. Perhaps fields that start with `[""` or `{""` and end with `""]` or `""}` could be detected, with a flag to turn off that behavior for weird text fields (or vice versa).",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/20/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 738115165,MDU6SXNzdWU3MzgxMTUxNjU=,200,sqlite-utils rows -c option,9599,closed,0,,6079500,1,2020-11-07T00:22:12Z,2020-11-07T00:28:48Z,2020-11-07T00:28:47Z,OWNER,,To let you specify the exact columns you want. Based on the `-c` option to `sqlite-utils search` in #192.,140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/200/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 738128913,MDU6SXNzdWU3MzgxMjg5MTM=,201,.search(columns=) and sqlite-utils search -c ... bug,9599,closed,0,,6079500,1,2020-11-07T01:27:26Z,2020-11-08T16:54:15Z,2020-11-08T16:54:15Z,OWNER,,"Both `table.search(columns=)` and the `sqlite-utils search -c` option do not work as expected - they always return both the `rowid` and the `rank` columns even if those have not been requested. This should be fixed before the 3.0 non-alpha release.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/201/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 738514367,MDU6SXNzdWU3Mzg1MTQzNjc=,202,sqlite-utils insert -f colname - for configuring full-text search,9599,closed,0,,,2,2020-11-08T17:30:09Z,2021-01-03T05:00:36Z,2021-01-03T05:00:27Z,OWNER,,"A mechanism for specifying columns that should be configured for full-text search as part of the initial data import: sqlite-utils insert mydb.db articles articles.csv --csv -f title -f body",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/202/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 760960559,MDU6SXNzdWU3NjA5NjA1NTk=,205,"sqlite3.OperationalError: near ""("": syntax error",765871,closed,0,,,2,2020-12-10T06:44:40Z,2020-12-10T19:18:22Z,2020-12-10T07:24:23Z,NONE,,"The sqlite version is 3.22.0 2018-01-22 18:45:57 0c55d179733b46d8d0ba4d88e01a25e10677046ee3da1d5b1581e86726f2alt1 sqlite-utils, version 3.0 It fails here: https://github.com/kaihendry/aws-partners-datasette/runs/1528432635?check_suite_focus=true I'm not sure where the problem is, since it works _fine locally_ on Archlinux system running 3.34.0 2020-12-01 16:14:00 a26b6597e3ae272231b96f9982c3bcc17ddec2f2b6eb4df06a224b91089fed5b https://github.com/kaihendry/aws-partners-datasette/blob/main/create-summary-view.sh Maybe I need to bump up from ubuntu-latest to ? ",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/205/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed 761915790,MDU6SXNzdWU3NjE5MTU3OTA=,206,sqlite-utils should suggest --csv if JSON parsing fails,9599,closed,0,,,4,2020-12-11T05:17:56Z,2021-10-30T15:52:17Z,2021-01-03T18:42:22Z,OWNER,,"``` ~ % gsutil cat gs://ossf-criticality-score/python_top_200.csv | sqlite-utils insert /tmp/crit.db crit - ... File ""/usr/local/Cellar/python@3.9/3.9.0_3/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/decoder.py"", line 337, in decode obj, end = self.raw_decode(s, idx=_w(s, 0).end()) File ""/usr/local/Cellar/python@3.9/3.9.0_3/Frameworks/Python.framework/Versions/3.9/lib/python3.9/json/decoder.py"", line 355, in raw_decode raise JSONDecodeError(""Expecting value"", s, err.value) from None json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0) ``` A nicer error message here would be one that says the JSON is invalid but suggests that maybe you could try `--csv`.",140912432,issue,,,"{""url"": ""https://api.github.com/repos/simonw/sqlite-utils/issues/206/reactions"", ""total_count"": 0, ""+1"": 0, ""-1"": 0, ""laugh"": 0, ""hooray"": 0, ""confused"": 0, ""heart"": 0, ""rocket"": 0, ""eyes"": 0}",,completed