issue_comments

Here's the diff where I tried to use .executemany() and ran into the lastrowid problem: diff diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index ef55976..7f85759 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -881,13 +881,10 @@ class Table: or_what=or_what, table=self.name, columns=", ".join("[{}]".format(c) for c in all_columns), - rows=", ".join( - """ + rows=""" ({placeholders}) """.format( - placeholders=", ".join(["?"] * len(all_columns)) - ) - for record in chunk + placeholders=", ".join(["?"] * len(all_columns)) ), ) values = [] @@ -902,15 +899,15 @@ class Table: extract_table = extracts[key] value = self.db[extract_table].lookup({"value": value}) record_values.append(value) - values.extend(record_values) + values.append(record_values) with self.db.conn: try: - result = self.db.conn.execute(sql, values) + result = self.db.conn.executemany(sql, values) except sqlite3.OperationalError as e: if alter and (" has no column " in e.args[0]): # Attempt to add any missing columns, then try again self.add_missing_columns(chunk) - result = self.db.conn.execute(sql, values) + result = self.db.conn.executemany(sql, values) else: raise self.last_rowid = result.lastrowid

The problem with .executemany() is it breaks lastrowid:

This read-only attribute provides the rowid of the last modified row. It is only set if you issued an INSERT or a REPLACE statement using the execute() method. For operations other than INSERT or REPLACE or when executemany() is called, lastrowid is set to None.

So I think I need to continue to use my existing way of executing bulk inserts (with a giant repeated INSERT INTO ... VALUES block) but ensure that I calculate the chunk size such that I don't ever try to pass more than 999 values at once.

I think the fix here is for me to switch to using executemany() - example from the Python docs: https://docs.python.org/3/library/sqlite3.html python purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), ('2006-04-06', 'SELL', 'IBM', 500, 53.00), ] c.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases)