Fix #611: ContactSchema now reflects what is stored in database

ctms/acoustic_service.py

@@ -12,7 +12,7 @@
 
 from ctms.background_metrics import BackgroundMetricService
 from ctms.config import re_trace_email
-from ctms.schemas import ContactSchema, NewsletterSchema
+from ctms.schemas import ContactTableSchema, NewsletterSchema
 
 # Start cherry-picked from django.utils.encoding
 _PROTECTED_TYPES = (
@@ -140,7 +140,7 @@ def __init__(
 
     def convert_ctms_to_acoustic(
         self,
-        contact: ContactSchema,
+        contact: ContactTableSchema,
         main_fields: set[str],
         newsletters_mapping: dict[str, str],
     ):
@@ -240,14 +240,12 @@ def _newsletter_converter(self, acoustic_main_table, contact, newsletters_mappin
             }
 
             if newsletter.name in newsletters_mapping:
-                newsletter_dict = newsletter.dict()
-                _today = datetime.date.today().isoformat()
-                newsletter_template["create_timestamp"] = newsletter_dict.get(
-                    "create_timestamp", _today
-                )
-                newsletter_template["update_timestamp"] = newsletter_dict.get(
-                    "update_timestamp", _today
-                )
+                newsletter_template[
+                    "create_timestamp"
+                ] = newsletter.create_timestamp.date().isoformat()
+                newsletter_template[
+                    "update_timestamp"
+                ] = newsletter.update_timestamp.date().isoformat()
                 newsletter_template["newsletter_name"] = newsletter.name
                 newsletter_template["newsletter_unsub_reason"] = newsletter.unsub_reason
                 _source = newsletter.source
@@ -430,7 +428,7 @@ def _insert_update_relational_table(self, table_name, rows):
 
     def attempt_to_upload_ctms_contact(
         self,
-        contact: ContactSchema,
+        contact: ContactTableSchema,
         main_fields: set[str],
         newsletters_mapping: dict[str, str],
     ):  # raises AcousticUploadError

ctms/app.py

@@ -70,7 +70,7 @@
     ContactInSchema,
     ContactPatchSchema,
     ContactPutSchema,
-    ContactSchema,
+    ContactTableSchema,
     CTMSBulkResponse,
     CTMSResponse,
     CTMSSingleResponse,
@@ -152,10 +152,10 @@ def get_email_or_404(db: Session, email_id) -> Email:
     return email
 
 
-def get_contact_or_404(db: Session, email_id) -> ContactSchema:
+def get_contact_or_404(db: Session, email_id) -> ContactTableSchema:
     """Get a contact by email_ID, or raise a 404 exception."""
     email = get_email_or_404(db, email_id)
-    return ContactSchema.from_email(email)
+    return ContactTableSchema.from_email(email)
 
 
 def all_ids(

ctms/bin/acoustic.py

@@ -27,7 +27,7 @@
 from ctms.database import SessionLocal
 from ctms.exception_capture import init_sentry
 from ctms.log import configure_logging
-from ctms.schemas.contact import ContactSchema
+from ctms.schemas.contact import ContactTableSchema
 
 
 def confirm(msg):
@@ -258,7 +258,7 @@ def do_dump(dbsession, contacts, output: TextIO):
     fieldnames = None
     writer = None
     for email in contacts:
-        contact = ContactSchema.from_email(email)
+        contact = ContactTableSchema.from_email(email)
         main_table_row, _, _ = service.convert_ctms_to_acoustic(
             contact, main_fields, newsletters_mapping
         )

ctms/crud.py

@@ -10,6 +10,10 @@
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.orm import Session, joinedload, load_only, selectinload
 
+from ctms.schemas.email import EmailSchema
+from ctms.schemas.newsletter import NewsletterTableSchema
+from ctms.schemas.waitlist import WaitlistTableSchema
+
 from .auth import hash_password
 from .backport_legacy_waitlists import format_legacy_vpn_relay_waitlist_input
 from .database import Base
@@ -37,7 +41,7 @@
     ApiClientSchema,
     ContactInSchema,
     ContactPutSchema,
-    ContactSchema,
+    ContactTableSchema,
     EmailInSchema,
     EmailPutSchema,
     FirefoxAccountsInSchema,
@@ -157,7 +161,7 @@ def get_bulk_contacts(
         .all()
     )
 
-    return [ContactSchema.from_email(email) for email in bulk_contacts]
+    return [ContactTableSchema.from_email(email) for email in bulk_contacts]
 
 
 def get_email(db: Session, email_id: UUID4) -> Optional[Email]:
@@ -168,12 +172,14 @@ def get_email(db: Session, email_id: UUID4) -> Optional[Email]:
     )
 
 
-def get_contact_by_email_id(db: Session, email_id: UUID4) -> Optional[ContactSchema]:
+def get_contact_by_email_id(
+    db: Session, email_id: UUID4
+) -> Optional[ContactTableSchema]:
     """Return a Contact object for a given email id"""
     email = get_email(db, email_id)
     if email is None:
         return None
-    return ContactSchema.from_email(email)
+    return ContactTableSchema.from_email(email)
 
 
 def get_contacts_by_any_id(
@@ -187,7 +193,7 @@ def get_contacts_by_any_id(
     amo_user_id: Optional[str] = None,
     fxa_id: Optional[str] = None,
     fxa_primary_email: Optional[str] = None,
-) -> List[ContactSchema]:
+) -> List[ContactTableSchema]:
     """
     Get all the data for multiple contacts by ID as a list of Contacts.
 
@@ -235,7 +241,7 @@ def get_contacts_by_any_id(
             fxa_primary_email_insensitive_comparator=fxa_primary_email
         )
     emails = cast(List[Email], statement.all())
-    return [ContactSchema.from_email(email) for email in emails]
+    return [ContactTableSchema.from_email(email) for email in emails]
 
 
 def _acoustic_sync_retry_query(db: Session):
@@ -369,7 +375,7 @@ def create_or_update_amo(db: Session, email_id: UUID4, amo: Optional[AddOnsInSch
     db.execute(stmt)
 
 
-def create_email(db: Session, email: EmailInSchema):
+def create_email(db: Session, email: EmailInSchema | EmailSchema):
     db_email = Email(**email.dict())
     db.add(db_email)
 
@@ -439,11 +445,14 @@ def create_or_update_mofo(
 
 
 def create_newsletter(
-    db: Session, email_id: UUID4, newsletter: NewsletterInSchema
+    db: Session, email_id: UUID4, newsletter: NewsletterInSchema | NewsletterTableSchema
 ) -> Optional[Newsletter]:
     if newsletter.is_default():
         return None
-    db_newsletter = Newsletter(email_id=email_id, **newsletter.dict())
+    # This is called from API input data with `NewsletterInSchema`, and from tests fixtures
+    # with `NewsletterTableSchema`
+    # Unlike `NewsletterTableSchema`, `NewsletterInSchema` has no `email_id` field.
+    db_newsletter = Newsletter(**{"email_id": email_id, **newsletter.dict()})
     db.add(db_newsletter)
     return db_newsletter
 
@@ -473,15 +482,19 @@ def create_or_update_newsletters(
 
 
 def create_waitlist(
-    db: Session, email_id: UUID4, waitlist: WaitlistInSchema
+    db: Session, email_id: UUID4, waitlist: WaitlistInSchema | WaitlistTableSchema
 ) -> Optional[Waitlist]:
     if waitlist.is_default():
         return None
-    if not isinstance(waitlist, WaitlistInSchema):
-        # Sample data are used as both input (`WaitlistInSchema`) and internal (`WaitlistSchema`)
-        # representations.
-        waitlist = WaitlistInSchema(**waitlist.dict())
-    db_waitlist = Waitlist(email_id=email_id, **waitlist.orm_dict())
+
+    # This is called from API input data with `WaitlistInSchema`, and from tests fixtures
+    # with `WaitlistTableSchema`
+    # Unlike `WaitlistTableSchema`, `WaitlistInSchema` has no `email_id`, `subscribed` fields.
+    attrs = {"email_id": email_id, **waitlist.dict()}
+    if "subscribed" in attrs:
+        del attrs["subscribed"]
+
+    db_waitlist = Waitlist(**attrs)
     db.add(db_waitlist)
     return db_waitlist
 
@@ -517,7 +530,10 @@ def create_or_update_waitlists(
 
 
 def create_contact(
-    db: Session, email_id: UUID4, contact: ContactInSchema, metrics: Optional[Dict]
+    db: Session,
+    email_id: UUID4,
+    contact: ContactInSchema | ContactTableSchema,
+    metrics: Optional[Dict],
 ):
     create_email(db, contact.email)
     if contact.amo:

ctms/schemas/__init__.py

@@ -5,7 +5,7 @@
     ContactInSchema,
     ContactPatchSchema,
     ContactPutSchema,
-    ContactSchema,
+    ContactTableSchema,
     CTMSBulkResponse,
     CTMSResponse,
     CTMSSingleResponse,
@@ -24,7 +24,12 @@
     UpdatedFirefoxAccountsInSchema,
 )
 from .mofo import MozillaFoundationInSchema, MozillaFoundationSchema
-from .newsletter import NewsletterInSchema, NewsletterSchema, UpdatedNewsletterInSchema
+from .newsletter import (
+    NewsletterInSchema,
+    NewsletterSchema,
+    NewsletterTableSchema,
+    UpdatedNewsletterInSchema,
+)
 from .product import ProductBaseSchema
 from .stripe_customer import (
     StripeCustomerCreateSchema,
@@ -68,6 +73,7 @@
     VpnWaitlistInSchema,
     WaitlistInSchema,
     WaitlistSchema,
+    WaitlistTableSchema,
 )
 from .web import (
     BadRequestResponse,