From acff85167db06e34b6d52a73e97e550b423e9561 Mon Sep 17 00:00:00 2001 From: Duncan Beevers Date: Tue, 7 Jun 2011 16:21:32 -0700 Subject: [PATCH 1/2] Use MySQL enums for hadron_action column to reduce necessary storage --- lib/large_hadron_migration.rb | 17 +++++++++-------- spec/large_hadron_migration_spec.rb | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/large_hadron_migration.rb b/lib/large_hadron_migration.rb index 9fe97d7b..ac6d68da 100644 --- a/lib/large_hadron_migration.rb +++ b/lib/large_hadron_migration.rb @@ -253,8 +253,8 @@ def self.with_master end end - def self.clone_table(source, dest, window = 0) - execute schema_sql(source, dest, window) + def self.clone_table(source, dest, window = 0, add_action_column = false) + execute schema_sql(source, dest, window, add_action_column) end def self.common_columns(t1, t2) @@ -262,11 +262,7 @@ def self.common_columns(t1, t2) end def self.clone_table_for_changes(table, journal_table) - clone_table(table, journal_table) - execute %Q{ - alter table %s - add column hadron_action varchar(15); - } % journal_table + clone_table(table, journal_table, 0, true) end def self.rename_tables(tables = {}) @@ -359,12 +355,17 @@ def self.replay_update_changes(table, journal_table, chunk_size = 10000, wait = # behavior with the latter where the auto_increment of the source table # got modified when updating the destination. # - def self.schema_sql(source, dest, window) + def self.schema_sql(source, dest, window, add_action_column = false) show_create(source).tap do |schema| schema.gsub!(/auto_increment=(\d+)/i) do "auto_increment=#{ $1.to_i + window }" end + if add_action_column + schema.sub!(/\) ENGINE=InnoDB/, + ", hadron_action ENUM('update', 'insert', 'delete') ) ENGINE=InnoDB") + end + schema.gsub!('CREATE TABLE `%s`' % source, 'CREATE TABLE `%s`' % dest) end end diff --git a/spec/large_hadron_migration_spec.rb b/spec/large_hadron_migration_spec.rb index 632e43fc..e32e1f01 100644 --- a/spec/large_hadron_migration_spec.rb +++ b/spec/large_hadron_migration_spec.rb @@ -99,7 +99,7 @@ end it "should create a table for triggered changes" do - truthiness_column "triggerme_changes", "hadron_action", "varchar" + truthiness_column "triggerme_changes", "hadron_action", "enum" end it "should trigger on insert" do From 371f41b879e39639e2a18b9d3ce79b3faaaabd77 Mon Sep 17 00:00:00 2001 From: Duncan Beevers Date: Wed, 8 Jun 2011 14:34:12 -0700 Subject: [PATCH 2/2] Index hadron_action --- lib/large_hadron_migration.rb | 4 ++-- spec/large_hadron_migration_spec.rb | 1 + spec/spec_helper.rb | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/large_hadron_migration.rb b/lib/large_hadron_migration.rb index ac6d68da..519ad9f4 100644 --- a/lib/large_hadron_migration.rb +++ b/lib/large_hadron_migration.rb @@ -362,8 +362,8 @@ def self.schema_sql(source, dest, window, add_action_column = false) end if add_action_column - schema.sub!(/\) ENGINE=InnoDB/, - ", hadron_action ENUM('update', 'insert', 'delete') ) ENGINE=InnoDB") + schema.sub!(/\) ENGINE=/, + ", hadron_action ENUM('update', 'insert', 'delete'), INDEX hadron_action (hadron_action) USING BTREE) ENGINE=") end schema.gsub!('CREATE TABLE `%s`' % source, 'CREATE TABLE `%s`' % dest) diff --git a/spec/large_hadron_migration_spec.rb b/spec/large_hadron_migration_spec.rb index e32e1f01..acec8e6d 100644 --- a/spec/large_hadron_migration_spec.rb +++ b/spec/large_hadron_migration_spec.rb @@ -100,6 +100,7 @@ it "should create a table for triggered changes" do truthiness_column "triggerme_changes", "hadron_action", "enum" + truthiness_index "triggerme_changes", "hadron_action", [ "hadron_action" ], false end it "should trigger on insert" do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index acf058e9..f9c97b3a 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -90,6 +90,21 @@ def truthiness_rows(table_name1, table_name2, offset = 0, limit = 1000) end + def truthiness_index(table, expected_index_name, indexed_columns, unique) + index = sql("SHOW INDEXES FROM #{table}").all_hashes.inject({}) do |a, part| + index_name = part['Key_name'] + a[index_name] ||= { 'unique' => '0' == part['Non_unique'], 'columns' => [] } + column_index = part['Seq_in_index'].to_i - 1 + a[index_name]['columns'][column_index] = part['Column_name'] + a + end[expected_index_name] + + flunk("no index named #{expected_index_name} found on #{table}") unless index + + index['columns'].should == indexed_columns + index['unique'].should == unique + end + end # Mock Rails Environment