refactoring

.github/workflows/testing.yml

@@ -17,14 +17,15 @@ jobs:
 
     strategy:
       fail-fast: true
+      max-parallel: 1
       matrix:
-        ruby-version: [ '3.0' ]
+        ruby-version: [ '2.7', '3.0', '3.2' ]
         clickhouse: [ '22.1' ]
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) in Docker
+      - name: Start ClickHouse ${{ matrix.clickhouse }}
         uses: isbang/compose-action@v1.5.1
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
@@ -51,14 +52,15 @@ jobs:
 
     strategy:
       fail-fast: true
+      max-parallel: 1
       matrix:
-        ruby-version: [ '3.0' ]
+        ruby-version: [ '2.7', '3.0', '3.2' ]
         clickhouse: [ '22.1' ]
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Start ClickHouse Cluster (version - ${{ matrix.clickhouse }}) in Docker
+      - name: Start ClickHouse Cluster ${{ matrix.clickhouse }}
         uses: isbang/compose-action@v1.5.1
         env:
           CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}

lib/clickhouse-activerecord.rb

@@ -5,15 +5,12 @@
 require 'core_extensions/active_record/internal_metadata'
 require 'core_extensions/active_record/relation'
 require 'core_extensions/active_record/schema_migration'
-
+require 'core_extensions/active_record/migration/command_recorder'
 require 'core_extensions/arel/nodes/select_core'
 require 'core_extensions/arel/nodes/select_statement'
 require 'core_extensions/arel/select_manager'
 require 'core_extensions/arel/table'
 
-require_relative '../core_extensions/active_record/migration/command_recorder'
-ActiveRecord::Migration::CommandRecorder.include CoreExtensions::ActiveRecord::Migration::CommandRecorder
-
 if defined?(Rails::Railtie)
   require 'clickhouse-activerecord/railtie'
   require 'clickhouse-activerecord/schema'
@@ -24,9 +21,10 @@
 
 module ClickhouseActiverecord
   def self.load
-    ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods)
+    ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata)
+    ActiveRecord::Migration::CommandRecorder.include(CoreExtensions::ActiveRecord::Migration::CommandRecorder)
     ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation)
-    ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods)
+    ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration)
 
     Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore)
     Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement)

lib/clickhouse-activerecord/tasks.rb

@@ -5,12 +5,12 @@ class Tasks
     delegate :connection, :establish_connection, to: ActiveRecord::Base
 
     def initialize(configuration)
-      @configuration = configuration.with_indifferent_access
+      @configuration = configuration
     end
 
     def create
       establish_master_connection
-      connection.create_database @configuration['database']
+      connection.create_database @configuration.database
     rescue ActiveRecord::StatementInvalid => e
       if e.cause.to_s.include?('already exists')
         raise ActiveRecord::DatabaseAlreadyExists
@@ -21,7 +21,7 @@ def create
 
     def drop
       establish_master_connection
-      connection.drop_database @configuration['database']
+      connection.drop_database @configuration.database
     end
 
     def purge
@@ -31,12 +31,21 @@ def purge
     end
 
     def structure_dump(*args)
-      tables = connection.execute("SHOW TABLES FROM #{@configuration['database']}")['data'].flatten
+      establish_master_connection
+
+      # get all tables
+      tables = connection.execute("SHOW TABLES FROM #{@configuration.database} WHERE name NOT LIKE '.inner_id.%'")['data'].flatten.map do |table|
+        next if %w[schema_migrations ar_internal_metadata].include?(table)
+        connection.show_create_table(table).gsub("#{@configuration.database}.", '')
+      end.compact
+
+      # sort view to last
+      tables.sort_by! {|table| table.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : 0}
 
+      # put to file
       File.open(args.first, 'w:utf-8') do |file|
         tables.each do |table|
-          next if table.match(/\.inner/)
-          file.puts connection.execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first).gsub("#{@configuration['database']}.", '') + ";\n\n"
+          file.puts table + ";\n\n"
         end
       end
     end

lib/core_extensions/active_record/internal_metadata.rb

@@ -1,55 +1,53 @@
 module CoreExtensions
   module ActiveRecord
     module InternalMetadata
-      module ClassMethods
-
-        def create_table
-          return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
-          return if table_exists? || !enabled?
-
-          key_options = connection.internal_string_options_for_primary_key
-          table_options = {
-            id: false,
-            options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key',
-            if_not_exists: true
-          }
-          full_config = connection.instance_variable_get(:@config) || {}
-
-          if full_config[:distributed_service_tables]
-            table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
-
-            distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
-          else
-            distributed_suffix = ''
-          end
-
-          connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
-            t.string :key, **key_options
-            t.string :value
-            t.timestamps
-          end
-        end
 
-        private
+      def create_table
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+        return if table_exists? || !enabled?
+
+        key_options = connection.internal_string_options_for_primary_key
+        table_options = {
+          id: false,
+          options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key',
+          if_not_exists: true
+        }
+        full_config = connection.instance_variable_get(:@config) || {}
 
-        def update_entry(key, new_value)
-          return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+        if full_config[:distributed_service_tables]
+          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)')
 
-          create_entry(key, new_value)
+          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
+        else
+          distributed_suffix = ''
         end
 
-        def select_entry(key)
-          return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
+          t.string :key, **key_options
+          t.string :value
+          t.timestamps
+        end
+      end
 
-          sm = ::Arel::SelectManager.new(arel_table)
-          sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/
-          sm.project(::Arel.star)
-          sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key)))
-          sm.order(arel_table[primary_key].asc)
-          sm.limit = 1
+      private
 
-          connection.select_one(sm, "#{self.class} Load")
-        end
+      def update_entry(key, new_value)
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+
+        create_entry(key, new_value)
+      end
+
+      def select_entry(key)
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+
+        sm = ::Arel::SelectManager.new(arel_table)
+        sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/
+        sm.project(::Arel.star)
+        sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key)))
+        sm.order(arel_table[primary_key].asc)
+        sm.limit = 1
+
+        connection.select_one(sm, "#{self.class} Load")
       end
     end
   end

lib/core_extensions/active_record/schema_migration.rb

@@ -1,47 +1,45 @@
 module CoreExtensions
   module ActiveRecord
     module SchemaMigration
-      module ClassMethods
 
-        def create_table
-          return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+      def create_table
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
 
-          return if table_exists?
+        return if table_exists?
 
-          version_options = connection.internal_string_options_for_primary_key
-          table_options = {
-            id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true
-          }
-          full_config = connection.instance_variable_get(:@config) || {}
+        version_options = connection.internal_string_options_for_primary_key
+        table_options = {
+          id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true
+        }
+        full_config = connection.instance_variable_get(:@config) || {}
 
-          if full_config[:distributed_service_tables]
-            table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)')
+        if full_config[:distributed_service_tables]
+          table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)')
 
-            distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
-          else
-            distributed_suffix = ''
-          end
+          distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}"
+        else
+          distributed_suffix = ''
+        end
 
-          connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
-            t.string :version, **version_options
-            t.column :active, 'Int8', null: false, default: '1'
-            t.datetime :ver, null: false, default: -> { 'now()' }
-          end
+        connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t|
+          t.string :version, **version_options
+          t.column :active, 'Int8', null: false, default: '1'
+          t.datetime :ver, null: false, default: -> { 'now()' }
         end
+      end
 
-        def delete_version(version)
-          return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+      def delete_version(version)
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
 
-          im = ::Arel::InsertManager.new(arel_table)
-          im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0)
-          connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version)
-        end
+        im = ::Arel::InsertManager.new(arel_table)
+        im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0)
+        connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version)
+      end
 
-        def all_versions
-          return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
+      def all_versions
+        return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter)
 
-          final.where(active: 1).order(:version).pluck(:version)
-        end
+        final.where(active: 1).order(:version).pluck(:version)
       end
     end
   end

lib/tasks/clickhouse.rake

@@ -15,15 +15,18 @@ namespace :clickhouse do
     # TODO: deprecated
     desc 'Load database schema'
     task load: %i[prepare_internal_metadata_table] do
+      puts 'Warning: `rake clickhouse:schema:load` is deprecated! Use `rake db:schema:load:clickhouse` instead'
       simple = ENV['simple'] || ARGV.any? { |a| a.include?('--simple') } ? '_simple' : nil
       ActiveRecord::Base.establish_connection(:clickhouse)
-      ActiveRecord::SchemaMigration.drop_table
+      connection = ActiveRecord::Tasks::DatabaseTasks.migration_connection
+      connection.schema_migration.drop_table
       load(Rails.root.join("db/clickhouse_schema#{simple}.rb"))
     end
 
     # TODO: deprecated
     desc 'Dump database schema'
     task dump: :environment do |_, args|
+      puts 'Warning: `rake clickhouse:schema:dump` is deprecated! Use `rake db:schema:dump:clickhouse` instead'
       simple = ENV['simple'] || args[:simple] || ARGV.any? { |a| a.include?('--simple') } ? '_simple' : nil
       filename = Rails.root.join("db/clickhouse_schema#{simple}.rb")
       File.open(filename, 'w:utf-8') do |file|
@@ -36,43 +39,38 @@ namespace :clickhouse do
   namespace :structure do
     desc 'Load database structure'
     task load: ['db:check_protected_environments'] do
-      config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
       ClickhouseActiverecord::Tasks.new(config).structure_load(Rails.root.join('db/clickhouse_structure.sql'))
     end
 
     desc 'Dump database structure'
     task dump: ['db:check_protected_environments'] do
-      config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
       ClickhouseActiverecord::Tasks.new(config).structure_dump(Rails.root.join('db/clickhouse_structure.sql'))
     end
   end
 
   desc 'Creates the database from DATABASE_URL or config/database.yml'
   task create: [] do
-    config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
-    ActiveRecord::Tasks::DatabaseTasks.create(config)
+    puts 'Warning: `rake clickhouse:create` is deprecated! Use `rake db:create:clickhouse` instead'
   end
 
   desc 'Drops the database from DATABASE_URL or config/database.yml'
   task drop: ['db:check_protected_environments'] do
-    config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
-    ActiveRecord::Tasks::DatabaseTasks.drop(config)
+    puts 'Warning: `rake clickhouse:drop` is deprecated! Use `rake db:drop:clickhouse` instead'
   end
 
   desc 'Empty the database from DATABASE_URL or config/database.yml'
   task purge: ['db:check_protected_environments'] do
-    config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
-    ActiveRecord::Tasks::DatabaseTasks.purge(config)
+    puts 'Warning: `rake clickhouse:purge` is deprecated! Use `rake db:reset:clickhouse` instead'
   end
 
   # desc 'Resets your database using your migrations for the current environment'
   task :reset  do
-    Rake::Task['clickhouse:purge'].execute
-    Rake::Task['clickhouse:migrate'].execute
+    puts 'Warning: `rake clickhouse:reset` is deprecated! Use `rake db:reset:clickhouse` instead'
   end
 
   desc 'Migrate the clickhouse database'
   task migrate: %i[prepare_schema_migration_table prepare_internal_metadata_table] do
+    puts 'Warning: `rake clickhouse:migrate` is deprecated! Use `rake db:migrate:clickhouse` instead'
     Rake::Task['db:migrate:clickhouse'].execute
     if File.exist? "#{Rails.root}/db/clickhouse_schema_simple.rb"
       Rake::Task['clickhouse:schema:dump'].execute(simple: true)
@@ -81,9 +79,14 @@ namespace :clickhouse do
 
   desc 'Rollback the clickhouse database'
   task rollback: %i[prepare_schema_migration_table prepare_internal_metadata_table] do
+    puts 'Warning: `rake clickhouse:rollback` is deprecated! Use `rake db:rollback:clickhouse` instead'
     Rake::Task['db:rollback:clickhouse'].execute
     if File.exist? "#{Rails.root}/db/clickhouse_schema_simple.rb"
       Rake::Task['clickhouse:schema:dump'].execute(simple: true)
     end
   end
+
+  def config
+    ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse')
+  end
 end