diff --git a/open_prices/common/openfoodfacts.py b/open_prices/common/openfoodfacts.py index 10aa06eb..d1d9527c 100644 --- a/open_prices/common/openfoodfacts.py +++ b/open_prices/common/openfoodfacts.py @@ -126,6 +126,7 @@ def import_product_db(flavor: Flavor = Flavor.off, batch_size: int = 1000) -> No from open_prices.products.models import Product print((f"Launching import_product_db ({flavor})")) + existing_product_codes = set(Product.objects.values_list("code", flat=True)) existing_product_flavor_codes = set( Product.objects.filter(source=flavor).values_list("code", flat=True) ) @@ -200,9 +201,10 @@ def import_product_db(flavor: Flavor = Flavor.off, batch_size: int = 1000) -> No product_dict = normalize_product_fields(product_dict) # Case 1: new OFF product (not in OP database) - if product_code not in existing_product_flavor_codes: + if product_code not in existing_product_codes: product_dict["code"] = product_code products_to_create_or_update.append(Product(**product_dict)) + existing_product_codes.add(product_code) added_count += 1 # Case 2: existing product (already in OP database) @@ -229,6 +231,7 @@ def import_product_db(flavor: Flavor = Flavor.off, batch_size: int = 1000) -> No ): Product.objects.bulk_create( products_to_create_or_update, + ignore_conflicts=True, update_conflicts=True, update_fields=OFF_FIELDS, unique_fields=["code"], @@ -239,6 +242,7 @@ def import_product_db(flavor: Flavor = Flavor.off, batch_size: int = 1000) -> No # final database update Product.objects.bulk_create( products_to_create_or_update, + ignore_conflicts=True, update_conflicts=True, update_fields=OFF_FIELDS, unique_fields=["code"],