Test durable resume, old task adoption and no duplicate writes after errors

This commit is contained in:
Sleeps
2026-10-08 11:09:16 +01:00
parent 9fadd68ed1
commit 8299c3f626
+122 -1
View File
@@ -266,6 +266,8 @@ class IncrementalCaptionSyncTests(unittest.TestCase):
self.assertEqual(count_caption_repair_documents(db_path), 3)
meili = MagicMock()
meili.index = "scenes"
meili.add_documents.side_effect = [101, 102]
progress = []
restored = repair_caption_documents(
db_path=db_path, meili=meili,
@@ -275,8 +277,10 @@ class IncrementalCaptionSyncTests(unittest.TestCase):
self.assertEqual(restored, 3)
self.assertEqual(meili.add_documents.call_count, 2)
self.assertTrue(all(
call.kwargs["wait"] for call in meili.add_documents.call_args_list
call.kwargs["wait"] is False
for call in meili.add_documents.call_args_list
))
self.assertEqual(meili.wait_for_task.call_count, 2)
docs = [
document
for call in meili.add_documents.call_args_list
@@ -296,6 +300,123 @@ class IncrementalCaptionSyncTests(unittest.TestCase):
self.assertIn("siglip2", doc["_vectors"])
self.assertEqual(len(doc["_vectors"]["siglip2"]), 2)
self.assertEqual(progress[-1]["documents_processed"], 3)
with connect(db_path) as con:
checkpoint = con.execute(
"SELECT * FROM caption_index_repair_checkpoints "
"WHERE index_uid = 'scenes'"
).fetchone()
self.assertEqual(checkpoint["completed_documents"], 3)
self.assertIsNone(checkpoint["pending_task_uid"])
def test_repair_resumes_pending_task_without_submitting_again(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db_path = Path(tmp) / "scenes.sqlite3"
self._seed(db_path, count=2)
meili = MagicMock()
meili.index = "scenes"
meili.add_documents.return_value = 710
meili.wait_for_task.side_effect = KeyboardInterrupt()
with self.assertRaises(KeyboardInterrupt):
repair_caption_documents(
db_path=db_path, meili=meili, batch_size=2
)
with connect(db_path) as con:
checkpoint = con.execute(
"SELECT * FROM caption_index_repair_checkpoints"
).fetchone()
self.assertEqual(checkpoint["pending_task_uid"], 710)
self.assertEqual(checkpoint["completed_documents"], 0)
meili.wait_for_task.side_effect = None
restored = repair_caption_documents(
db_path=db_path, meili=meili, batch_size=2
)
self.assertEqual(restored, 2)
self.assertEqual(meili.add_documents.call_count, 1)
self.assertEqual(meili.wait_for_task.call_count, 2)
with connect(db_path) as con:
checkpoint = con.execute(
"SELECT * FROM caption_index_repair_checkpoints"
).fetchone()
self.assertIsNone(checkpoint["pending_task_uid"])
self.assertEqual(checkpoint["completed_documents"], 2)
# Subsequent runs skip everything already confirmed.
self.assertEqual(
repair_caption_documents(
db_path=db_path, meili=meili, batch_size=2
), 0
)
self.assertEqual(meili.add_documents.call_count, 1)
def test_adopt_old_first_batch_avoids_duplicating_task(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db_path = Path(tmp) / "scenes.sqlite3"
self._seed(db_path, count=2)
meili = MagicMock()
meili.index = "scenes"
meili.get_task.return_value = {
"uid": 41758,
"indexUid": "scenes",
"type": "documentAdditionOrUpdate",
"status": "succeeded",
"details": {"receivedDocuments": 2},
}
restored = repair_caption_documents(
db_path=db_path,
meili=meili,
batch_size=2,
resume_task_uid=41758,
)
self.assertEqual(restored, 2)
meili.add_documents.assert_not_called()
meili.wait_for_task.assert_called_once()
with connect(db_path) as con:
row = con.execute(
"SELECT * FROM caption_index_repair_checkpoints"
).fetchone()
self.assertEqual(row["completed_documents"], 2)
def test_pending_task_error_is_not_silently_duplicated(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
db_path = Path(tmp) / "scenes.sqlite3"
self._seed(db_path, count=1)
meili = MagicMock()
meili.index = "scenes"
meili.add_documents.return_value = 512
meili.wait_for_task.side_effect = RuntimeError(
"API temporarily unavailable"
)
meili.get_task.return_value = {
"status": "processing",
}
with self.assertRaisesRegex(
RuntimeError, "API temporarily unavailable"
):
repair_caption_documents(
db_path=db_path, meili=meili, batch_size=1
)
with connect(db_path) as con:
row = con.execute(
"SELECT * FROM caption_index_repair_checkpoints"
).fetchone()
self.assertEqual(row["pending_task_uid"], 512)
self.assertEqual(row["completed_documents"], 0)
meili.wait_for_task.side_effect = None
self.assertEqual(
repair_caption_documents(
db_path=db_path, meili=meili, batch_size=1
), 1
)
self.assertEqual(meili.add_documents.call_count, 1)
if __name__ == "__main__":