fix: keep offset2id updated when deleting in sqlite (#471)

docarray/array/storage/sqlite/getsetdel.py

@@ -13,6 +13,7 @@ class GetSetDelMixin(BaseGetSetDelMixin):
 
     def _del_doc_by_id(self, _id: str):
         self._sql(f'DELETE FROM {self._table_name} WHERE doc_id=?', (_id,))
+        self._save_offset2ids()
         self._commit()
 
     def _set_doc_by_id(self, _id: str, value: 'Document'):
@@ -47,6 +48,7 @@ def _del_docs_by_ids(self, ids: str) -> Iterable['Document']:
             f"DELETE FROM {self._table_name} WHERE doc_id in ({','.join(['?'] * len(ids))})",
             ids,
         )
+        self._save_offset2ids()
         self._commit()
 
     def _load_offset2ids(self):
@@ -64,3 +66,11 @@ def _save_offset2ids(self):
                 (offset, doc_id),
             )
         self._commit()
+
+    def _del_docs(self, ids):
+        super()._del_docs(ids)
+        self._save_offset2ids()
+
+    def _del_doc_by_offset(self, offset: int):
+        super()._del_doc_by_offset(offset)
+        self._save_offset2ids()

tests/unit/array/test_sequence.py

@@ -103,3 +103,59 @@ def test_context_manager_from_disk(storage, config, start_storage, tmpdir, tmpfi
 
     del da
     del da2
+
+
+@pytest.mark.parametrize(
+    'storage, config',
+    [
+        ('memory', None),
+        ('weaviate', {'n_dim': 3, 'distance': 'l2-squared'}),
+        ('annlite', {'n_dim': 3, 'metric': 'Euclidean'}),
+        ('qdrant', {'n_dim': 3, 'distance': 'euclidean'}),
+        ('elasticsearch', {'n_dim': 3, 'distance': 'l2_norm'}),
+        ('sqlite', dict()),
+    ],
+)
+@pytest.mark.parametrize(
+    'index', [1, '1', slice(1, 2), [1], [False, True, False, False, False]]
+)
+def test_del_and_append(index, storage, config):
+    da = DocumentArray(storage=storage, config=config)
+
+    with da:
+        da.extend([Document(id=str(i)) for i in range(5)])
+    with da:
+        del da[index]
+        da.append(Document(id='new'))
+
+    assert da[:, 'id'] == ['0', '2', '3', '4', 'new']
+
+
+@pytest.mark.parametrize(
+    'index', [1, '1', slice(1, 2), [1], [False, True, False, False, False]]
+)
+@pytest.mark.parametrize(
+    'storage, config',
+    [
+        ('memory', None),
+        ('weaviate', {'n_dim': 3, 'distance': 'l2-squared'}),
+        ('annlite', {'n_dim': 3, 'metric': 'Euclidean'}),
+        ('qdrant', {'n_dim': 3, 'distance': 'euclidean'}),
+        ('elasticsearch', {'n_dim': 3, 'distance': 'l2_norm'}),
+        ('sqlite', dict()),
+    ],
+)
+def test_set_and_append(index, storage, config):
+    da = DocumentArray(storage=storage, config=config)
+
+    with da:
+        da.extend([Document(id=str(i)) for i in range(5)])
+    with da:
+        da[index] = (
+            Document(id='new')
+            if isinstance(index, int) or isinstance(index, str)
+            else [Document(id='new')]
+        )
+        da.append(Document(id='new_new'))
+
+    assert da[:, 'id'] == ['0', 'new', '2', '3', '4', 'new_new']