The problem is that any exceptions occur inside this block, which is run by db.execute_write_fn(insert_docs, block=False) and hence can't interrupt a current page request.

datasette-upload-csvs/datasette_upload_csvs/__init__.py

Lines 92 to 125 in 8127afc

    
           def insert_docs(conn): 
        
               database = sqlite_utils.Database(conn) 
        
               # TODO: Support other encodings: 
        
               reader = csv_std.reader(codecs.iterdecode(csv.file, "utf-8")) 
        
               headers = next(reader) 
        
               docs = (dict(zip(headers, row)) for row in reader) 
        
               i = 0 
        
               def docs_with_progress(): 
        
                   nonlocal i 
        
                   for doc in docs: 
        
                       i += 1 
        
                       yield doc 
        
                       if i % 10 == 0: 
        
                           database["_csv_progress_"].update( 
        
                               task_id, 
        
                               { 
        
                                   "rows_done": i, 
        
                                   "bytes_done": csv.file.tell(), 
        
                               }, 
        
                           ) 
        
               database[filename].insert_all(docs_with_progress(), alter=True, batch_size=100) 
        
               database["_csv_progress_"].update( 
        
                   task_id, 
        
                   { 
        
                       "rows_done": i, 
        
                       "bytes_done": total_size, 
        
                       "completed": str(datetime.datetime.utcnow()), 
        
                   }, 
        
               ) 
        
               return database[filename].count

Instead, it should write something to the _csv_progress_ table such that the polling JavaScript can detect that an error has occurred.

	def insert_docs(conn):
	database = sqlite_utils.Database(conn)
	# TODO: Support other encodings:
	reader = csv_std.reader(codecs.iterdecode(csv.file, "utf-8"))
	headers = next(reader)

	docs = (dict(zip(headers, row)) for row in reader)

	i = 0

	def docs_with_progress():
	nonlocal i
	for doc in docs:
	i += 1
	yield doc
	if i % 10 == 0:
	database["_csv_progress_"].update(
	task_id,
	{
	"rows_done": i,
	"bytes_done": csv.file.tell(),
	},
	)

	database[filename].insert_all(docs_with_progress(), alter=True, batch_size=100)
	database["_csv_progress_"].update(
	task_id,
	{
	"rows_done": i,
	"bytes_done": total_size,
	"completed": str(datetime.datetime.utcnow()),
	},
	)
	return database[filename].count

Uh oh!

Don't hide errors in imports from the user #24

Description

Metadata

Metadata

Assignees

Labels

Projects

Milestone

Relationships

Development

Issue actions