Verified Commit b7713ff2 authored by guglielmo's avatar guglielmo
Browse files

logger shows progress of loading each 1000 operations

parent 59b98a0f
Pipeline #3535 failed with stage
......@@ -340,12 +340,14 @@ class DjangoUpdateOrCreateLoader(Loader):
records = self.etl.processed_data.to_dict(orient="records")
self.logger.debug("Attempting to import {0} records.".format(len(records)))
c = 0
for record in records:
n = len(records)
for i, record in enumerate(records, start=1):
defaults = {k: v for k, v in record.items() if k in fields_to_update}
new_values = {k: v for k, v in record.items() if k not in defaults}
obj, created = self.django_model.objects.update_or_create(defaults=defaults, **new_values)
c = c + (1 if created else 0)
if i % 1000 == 0:
self.logger.info(f"{i}/{n}")
table = self.django_model._meta.db_table
n_created = c
n_updated = len(records) - c
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment