Skip to content

Commit

Permalink
batch index to have single batch mode for source/collections of an or…
Browse files Browse the repository at this point in the history
…g/user
  • Loading branch information
snyaggarwal committed Nov 17, 2023
1 parent 3c5d1af commit e61d65d
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 5 deletions.
9 changes: 6 additions & 3 deletions core/common/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -177,10 +177,13 @@ def get_exact_or_criteria(attr, values):
return criteria

@staticmethod
def batch_index(queryset, document):
def batch_index(queryset, document, single_batch=False):
if not get(settings, 'TEST_MODE'):
for batch in queryset.iterator(chunk_size=500):
document().update(batch, parallel=True)
if single_batch:

This comment has been minimized.

Copy link
@rkorytkowski

rkorytkowski Dec 8, 2023

Contributor

@snyaggarwal Could you please explain what's the single_batch for? Maybe add a method doc.

This comment has been minimized.

Copy link
@snyaggarwal

snyaggarwal Dec 12, 2023

Author Contributor

The only place its used at is for marking org/user's source/collections is_active True/False based on owner is_active

instance.batch_index(instance.source_set, SourceDocument, True)

This comment has been minimized.

Copy link
@snyaggarwal

snyaggarwal Dec 12, 2023

Author Contributor

Can remove this

document().update(queryset, parallel=True)
else:
for batch in queryset.iterator(chunk_size=500):
document().update(batch, parallel=True)

@staticmethod
@transaction.atomic
Expand Down
4 changes: 2 additions & 2 deletions core/common/signals.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ def propagate_owner_status(sender, instance=None, created=False, **kwargs): # p

if updated_sources:
from core.sources.documents import SourceDocument
instance.batch_index(instance.source_set, SourceDocument)
instance.batch_index(instance.source_set, SourceDocument, True)
if updated_collections:
from core.collections.documents import CollectionDocument
instance.batch_index(instance.collection_set, CollectionDocument)
instance.batch_index(instance.collection_set, CollectionDocument, True)

0 comments on commit e61d65d

Please sign in to comment.