From 99112e5bb132ef9f57c93bde44e95def42af00a1 Mon Sep 17 00:00:00 2001 From: Raphael Odini Date: Sun, 15 Sep 2024 16:06:14 +0200 Subject: [PATCH] fix(Django): fix daily import script (avoid ON CONFLICT DO UPDATE error. duplicate). ref #376 --- open_prices/common/openfoodfacts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/open_prices/common/openfoodfacts.py b/open_prices/common/openfoodfacts.py index 10aa06eb..d1d9527c 100644 --- a/open_prices/common/openfoodfacts.py +++ b/open_prices/common/openfoodfacts.py @@ -126,6 +126,7 @@ def import_product_db(flavor: Flavor = Flavor.off, batch_size: int = 1000) -> No from open_prices.products.models import Product print((f"Launching import_product_db ({flavor})")) + existing_product_codes = set(Product.objects.values_list("code", flat=True)) existing_product_flavor_codes = set( Product.objects.filter(source=flavor).values_list("code", flat=True) ) @@ -200,9 +201,10 @@ def import_product_db(flavor: Flavor = Flavor.off, batch_size: int = 1000) -> No product_dict = normalize_product_fields(product_dict) # Case 1: new OFF product (not in OP database) - if product_code not in existing_product_flavor_codes: + if product_code not in existing_product_codes: product_dict["code"] = product_code products_to_create_or_update.append(Product(**product_dict)) + existing_product_codes.add(product_code) added_count += 1 # Case 2: existing product (already in OP database) @@ -229,6 +231,7 @@ def import_product_db(flavor: Flavor = Flavor.off, batch_size: int = 1000) -> No ): Product.objects.bulk_create( products_to_create_or_update, + ignore_conflicts=True, update_conflicts=True, update_fields=OFF_FIELDS, unique_fields=["code"], @@ -239,6 +242,7 @@ def import_product_db(flavor: Flavor = Flavor.off, batch_size: int = 1000) -> No # final database update Product.objects.bulk_create( products_to_create_or_update, + ignore_conflicts=True, update_conflicts=True, update_fields=OFF_FIELDS, unique_fields=["code"],