feat(event): Add logical separation between application and events da…

…tabases (#1349)

* feat(events): Update database config to separate events from application DB
* feat(event): Denormalizer queries
* feat(event): Use only ids when referencing events

Gemfile

@@ -85,6 +85,7 @@ group :development, :test do
 end
 
 group :test do
+  gem 'database_cleaner-active_record'
   gem 'rspec-graphql_matchers'
   gem 'shoulda-matchers'
 end

Gemfile.lock

@@ -126,6 +126,10 @@ GEM
     crack (0.4.5)
       rexml
     crass (1.0.6)
+    database_cleaner-active_record (2.1.0)
+      activerecord (>= 5.a)
+      database_cleaner-core (~> 2.0.0)
+    database_cleaner-core (2.0.1)
     date (3.3.3)
     debug (1.6.2)
       irb (>= 1.3.6)
@@ -474,6 +478,7 @@ DEPENDENCIES
   clockwork-test
   coffee-rails
   countries
+  database_cleaner-active_record
   debug
   discard (~> 1.2)
   dotenv

app/graphql/resolvers/events_resolver.rb

@@ -15,25 +15,9 @@ class EventsResolver < GraphQL::Schema::Resolver
     def resolve(page: nil, limit: nil)
       validate_organization!
 
-      current_organization.events
-        .order(timestamp: :desc)
+      Event.where(organization_id: current_organization.id)
         .includes(:customer)
-        .joins('LEFT OUTER JOIN billable_metrics ON billable_metrics.code = events.code')
-        .where(billable_metrics: { deleted_at: nil })
-        .where(
-          [
-            'billable_metrics.organization_id = ?',
-            'billable_metrics.organization_id IS NULL',
-          ].join(' OR '),
-          current_organization.id,
-        )
-        .select(
-          [
-            'events.*',
-            'billable_metrics.name as billable_metric_name',
-            'billable_metrics.field_name as billable_metric_field_name',
-          ].join(','),
-        )
+        .order(timestamp: :desc)
         .page(page)
         .per(limit)
     end

app/graphql/types/events/object.rb

@@ -48,18 +48,23 @@ def payload
       end
 
       def match_billable_metric
-        object.billable_metric_name.present?
+        object.billable_metric.present?
       end
 
       def match_custom_field
-        return true if object.billable_metric_field_name.blank?
+        return true if object.billable_metric.blank?
+        return true if object.billable_metric.field_name.blank?
 
-        object.properties.key?(object.billable_metric_field_name)
+        object.properties.key?(object.billable_metric.field_name)
       end
 
       def customer_timezone
         object.customer.applicable_timezone
       end
+
+      def billable_metric_name
+        object.billable_metric&.name
+      end
     end
   end
 end

app/jobs/billable_metrics/delete_events_job.rb

@@ -9,9 +9,10 @@ def perform(metric)
 
       deleted_at = Time.current
 
-      Event.joins(subscription: [:plan]).where(
-        code: metric.code,
-        plan: { id: Charge.with_discarded.where(billable_metric_id: metric.id).pluck(:plan_id) },
+      Event.where(
+        subscription_id: Charge.with_discarded
+          .where(billable_metric_id: metric.id)
+          .joins(plan: :subscriptions).pluck('subscriptions.id'),
       ).update_all(deleted_at:) # rubocop:disable Rails/SkipsModelValidations
 
       metric.quantified_events.update_all(deleted_at:) # rubocop:disable Rails/SkipsModelValidations

app/models/event.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class Event < ApplicationRecord
+class Event < EventsRecord
   include Discard::Model
   self.discard_column = :deleted_at
 
@@ -27,4 +27,8 @@ def api_client
   def ip_address
     metadata['ip_address']
   end
+
+  def billable_metric
+    @billable_metric ||= organization.billable_metrics.find_by(code:)
+  end
 end

app/models/events_record.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class EventsRecord < ApplicationRecord
+  self.abstract_class = true
+
+  connects_to database: { writing: :primary, reading: :events }
+end

app/services/billable_metrics/aggregations/base_service.rb

@@ -50,9 +50,13 @@ def events_scope(from_datetime:, to_datetime:)
       end
 
       def recurring_events_scope(to_datetime:, from_datetime: nil)
-        events = customer.events
-          .joins(:subscription)
-          .where(subscription: { external_id: subscription.external_id })
+        subscription_ids = customer.subscriptions
+          .where(external_id: subscription.external_id)
+          .pluck(:id)
+
+        events = Event
+          .where(customer_id: customer.id)
+          .where(subscription_id: subscription_ids)
           .where(code: billable_metric.code)
           .to_datetime(to_datetime)
         events = events.from_datetime(from_datetime) unless from_datetime.nil?

config/database.yml

@@ -2,23 +2,49 @@ default: &default
   adapter: postgresql
 
 development:
-  <<: *default
-  host: localhost
-  username: lago
-  password: changeme
-  database: lago
-  port: 5432
+  primary:
+    <<: *default
+    host: localhost
+    username: lago
+    password: changeme
+    database: lago
+    port: 5432
+  events:
+    <<: *default
+    host: localhost
+    username: lago
+    password: changeme
+    database: lago
+    port: 5432
+    database_tasks: false
 
 test:
-  <<: *default
-  url: <%= ENV['DATABASE_TEST_URL'].presence || ENV['DATABASE_URL'] %>
+  primary:
+    <<: *default
+    url: <%= ENV['DATABASE_TEST_URL'].presence || ENV['DATABASE_URL'] %>
+  events:
+    <<: *default
+    url: <%= ENV['DATABASE_TEST_URL'].presence || ENV['DATABASE_URL'] %>
+    database_tasks: false
 
 staging:
-  <<: *default
-  url: <%= ENV['DATABASE_URL'] %>
+  primary:
+    <<: *default
+    url: <%= ENV['DATABASE_URL'] %>
+  events:
+    <<: *default
+    url: <%= ENV['DATABASE_URL'] %>
+    database_tasks: false
 
 production:
-  <<: *default
-  url: <%= ENV['DATABASE_URL'] %>
-  pool: <%= ENV.fetch('DATABASE_POOL', 10) %>
-  prepared_statements: <%= ENV.fetch('DATABASE_PREPARED_STATEMENTS', true) %>
+  primary:
+    <<: *default
+    url: <%= ENV['DATABASE_URL'] %>
+    pool: <%= ENV.fetch('DATABASE_POOL', 10) %>
+    prepared_statements: <%= ENV.fetch('DATABASE_PREPARED_STATEMENTS', true) %>
+  events:
+    <<: *default
+    url: <%= ENV['DATABASE_URL'] %>
+    pool: <%= ENV.fetch('DATABASE_POOL', 10) %>
+    prepared_statements: <%= ENV.fetch('DATABASE_PREPARED_STATEMENTS', true) %>
+    database_tasks: false

spec/factories/events.rb

@@ -2,9 +2,9 @@
 
 FactoryBot.define do
   factory :event do
-    organization
-    customer
-    subscription
+    organization_id { create(:organization).id }
+    customer_id { create(:customer).id }
+    subscription_id { create(:subscription).id }
 
     transaction_id { SecureRandom.uuid }
     code { Faker::Name.name.underscore }

spec/graphql/resolvers/events_resolver_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rails_helper'
 
-RSpec.describe Resolvers::EventsResolver, type: :graphql do
+RSpec.describe Resolvers::EventsResolver, type: :graphql, transaction: false do
   let(:query) do
     <<~GQL
       query {

spec/jobs/billable_metrics/delete_events_job_spec.rb

@@ -2,13 +2,13 @@
 
 require 'rails_helper'
 
-RSpec.describe BillableMetrics::DeleteEventsJob, type: :job do
+RSpec.describe BillableMetrics::DeleteEventsJob, type: :job, transaction: false do
   let(:billable_metric) { create(:billable_metric, :deleted) }
   let(:subscription) { create(:subscription) }
 
   it 'deletes related events' do
     create(:standard_charge, plan: subscription.plan, billable_metric:)
-    event = create(:event, code: billable_metric.code, subscription:)
+    event = create(:event, code: billable_metric.code, subscription_id: subscription.id)
     quantified_event = create(:quantified_event, billable_metric:)
 
     freeze_time do

spec/models/invoice_spec.rb

@@ -187,7 +187,7 @@
     let(:invoice) { invoice_subscription.invoice }
     let(:subscription) { invoice_subscription.subscription }
     let(:timestamp) { DateTime.parse('2023-07-25 00:00:00 UTC') }
-    let(:event) { create(:event, subscription:, timestamp:) }
+    let(:event) { create(:event, subscription_id: subscription.id, timestamp:) }
     let(:billable_metric) { create(:sum_billable_metric, organization: subscription.organization, recurring: true) }
     let(:fee) { create(:charge_fee, subscription:, invoice:, charge:, pay_in_advance_event_id: event.id) }
     let(:charge) do

spec/rails_helper.rb

@@ -7,6 +7,8 @@
 require 'spec_helper'
 require 'simplecov'
 
+DatabaseCleaner.allow_remote_database_url = true
+
 SimpleCov.start do
   enable_coverage :branch
 
@@ -39,6 +41,7 @@
   puts e.to_s.strip
   exit 1
 end
+
 RSpec.configure do |config|
   config.include FactoryBot::Syntax::Methods
   config.include GraphQLHelper, type: :graphql
@@ -54,7 +57,7 @@
   # If you're not using ActiveRecord, or you'd prefer not to run each of your
   # examples within a transaction, remove the following line or assign false
   # instead of true.
-  config.use_transactional_fixtures = true
+  config.use_transactional_fixtures = false
 
   # You can uncomment this line to turn off ActiveRecord support entirely.
   # config.use_active_record = false

spec/scenarios/billable_metrics/weighted_sum_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rails_helper'
 
-describe 'Aggregation - Weighted Sum Scenarios', :scenarios, type: :request do
+describe 'Aggregation - Weighted Sum Scenarios', :scenarios, type: :request, transaction: false do
   let(:organization) { create(:organization, webhook_url: nil) }
   let(:customer) { create(:customer, organization:) }
 

spec/scenarios/invoices_spec.rb

@@ -428,8 +428,8 @@
 
       ### 17 Dec: Create event + refresh.
       travel_to(DateTime.new(2022, 12, 17)) do
-        create(:event, subscription:, code: metric.code)
-        create(:event, subscription:, code: metric.code)
+        create(:event, subscription_id: subscription.id, code: metric.code)
+        create(:event, subscription_id: subscription.id, code: metric.code)
 
         expect {
           refresh_invoice(subscription_invoice)
@@ -512,7 +512,7 @@
 
       ### 17 Dec: Create event + refresh.
       travel_to(DateTime.new(2022, 12, 17)) do
-        create(:event, subscription:, code: metric.code)
+        create(:event, subscription_id: subscription.id, code: metric.code)
 
         expect {
           refresh_invoice(subscription_invoice)
@@ -593,7 +593,7 @@
 
       ### 16 Dec: Create event + refresh.
       travel_to(DateTime.new(2022, 12, 16)) do
-        create(:event, subscription:, code: metric.code)
+        create(:event, subscription_id: subscription.id, code: metric.code)
 
         # Paid in advance invoice amount does not change.
         expect {
@@ -603,7 +603,7 @@
 
       ### 17 Dec: Create event + refresh.
       travel_to(DateTime.new(2022, 12, 17)) do
-        create(:event, subscription:, code: metric.code)
+        create(:event, subscription_id: subscription.id, code: metric.code)
 
         # Paid in advance invoice amount does not change.
         expect {
@@ -620,7 +620,7 @@
         expect(new_invoice.total_amount_cents).to eq(1440) # (1000 + 200) * 1.2
 
         # Create event for Dec 18.
-        create(:event, subscription:, timestamp: DateTime.new(2022, 12, 18), code: metric.code)
+        create(:event, subscription_id: subscription.id, timestamp: DateTime.new(2022, 12, 18), code: metric.code)
 
         # Paid in advance invoice amount does not change.
         expect {

spec/scenarios/pay_in_advance_charges_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rails_helper'
 
-describe 'Pay in advance charges Scenarios', :scenarios, type: :request do
+describe 'Pay in advance charges Scenarios', :scenarios, type: :request, transaction: false do
   let(:organization) { create(:organization, webhook_url: nil) }
   let(:customer) { create(:customer, organization:) }
 

spec/serializers/v1/fee_serializer_spec.rb

@@ -100,7 +100,15 @@
     let(:plan) { create(:plan, organization:) }
     let(:subscription) { create(:subscription, customer:, organization:, plan:) }
     let(:charge) { create(:standard_charge, :pay_in_advance, plan:) }
-    let(:event) { create(:event, subscription:, organization:, customer:) }
+
+    let(:event) do
+      create(
+        :event,
+        subscription_id: subscription.id,
+        organization_id: organization.id,
+        customer_id: customer.id,
+      )
+    end
 
     let(:fee) do
       create(:charge_fee, pay_in_advance: true, subscription:, charge:, pay_in_advance_event_id: event.id)

spec/services/billable_metrics/aggregations/count_service_spec.rb

@@ -125,7 +125,7 @@
   end
 
   context 'when pay_in_advance aggregation' do
-    let(:pay_in_advance_event) { create(:event, subscription:, customer:) }
+    let(:pay_in_advance_event) { create(:event, subscription_id: subscription.id, customer_id: customer.id) }
 
     it 'assigns an pay_in_advance aggregation' do
       result = count_service.aggregate

spec/services/billable_metrics/aggregations/sum_service_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rails_helper'
 
-RSpec.describe BillableMetrics::Aggregations::SumService, type: :service do
+RSpec.describe BillableMetrics::Aggregations::SumService, type: :service, transaction: false do
   subject(:sum_service) do
     described_class.new(
       billable_metric:,

spec/services/billable_metrics/aggregations/unique_count_service_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rails_helper'
 
-RSpec.describe BillableMetrics::Aggregations::UniqueCountService, type: :service do
+RSpec.describe BillableMetrics::Aggregations::UniqueCountService, type: :service, transaction: false do
   subject(:count_service) do
     described_class.new(
       billable_metric:,

spec/services/billable_metrics/aggregations/weighted_sum_service_spec.rb

@@ -2,7 +2,7 @@
 
 require 'rails_helper'
 
-RSpec.describe BillableMetrics::Aggregations::WeightedSumService, type: :service do
+RSpec.describe BillableMetrics::Aggregations::WeightedSumService, type: :service, transaction: false do
   subject(:aggregator) do
     described_class.new(
       billable_metric:,