diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d42e8e02..6cd96c8f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -17,14 +17,15 @@ jobs: strategy: fail-fast: true + max-parallel: 1 matrix: - ruby-version: [ '3.0' ] + ruby-version: [ '2.7', '3.0', '3.2' ] clickhouse: [ '22.1' ] steps: - uses: actions/checkout@v4 - - name: Start ClickHouse (version - ${{ matrix.clickhouse }}) in Docker + - name: Start ClickHouse ${{ matrix.clickhouse }} uses: isbang/compose-action@v1.5.1 env: CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} @@ -51,14 +52,15 @@ jobs: strategy: fail-fast: true + max-parallel: 1 matrix: - ruby-version: [ '3.0' ] + ruby-version: [ '2.7', '3.0', '3.2' ] clickhouse: [ '22.1' ] steps: - uses: actions/checkout@v4 - - name: Start ClickHouse Cluster (version - ${{ matrix.clickhouse }}) in Docker + - name: Start ClickHouse Cluster ${{ matrix.clickhouse }} uses: isbang/compose-action@v1.5.1 env: CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} diff --git a/lib/clickhouse-activerecord.rb b/lib/clickhouse-activerecord.rb index 4eeb4158..40ce7ef7 100644 --- a/lib/clickhouse-activerecord.rb +++ b/lib/clickhouse-activerecord.rb @@ -5,15 +5,12 @@ require 'core_extensions/active_record/internal_metadata' require 'core_extensions/active_record/relation' require 'core_extensions/active_record/schema_migration' - +require 'core_extensions/active_record/migration/command_recorder' require 'core_extensions/arel/nodes/select_core' require 'core_extensions/arel/nodes/select_statement' require 'core_extensions/arel/select_manager' require 'core_extensions/arel/table' -require_relative '../core_extensions/active_record/migration/command_recorder' -ActiveRecord::Migration::CommandRecorder.include CoreExtensions::ActiveRecord::Migration::CommandRecorder - if defined?(Rails::Railtie) require 'clickhouse-activerecord/railtie' require 'clickhouse-activerecord/schema' @@ -24,9 +21,10 @@ module ClickhouseActiverecord def self.load - ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata::ClassMethods) + ActiveRecord::InternalMetadata.prepend(CoreExtensions::ActiveRecord::InternalMetadata) + ActiveRecord::Migration::CommandRecorder.include(CoreExtensions::ActiveRecord::Migration::CommandRecorder) ActiveRecord::Relation.prepend(CoreExtensions::ActiveRecord::Relation) - ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration::ClassMethods) + ActiveRecord::SchemaMigration.prepend(CoreExtensions::ActiveRecord::SchemaMigration) Arel::Nodes::SelectCore.prepend(CoreExtensions::Arel::Nodes::SelectCore) Arel::Nodes::SelectStatement.prepend(CoreExtensions::Arel::Nodes::SelectStatement) diff --git a/lib/clickhouse-activerecord/tasks.rb b/lib/clickhouse-activerecord/tasks.rb index e3635992..b8a4842c 100644 --- a/lib/clickhouse-activerecord/tasks.rb +++ b/lib/clickhouse-activerecord/tasks.rb @@ -5,12 +5,12 @@ class Tasks delegate :connection, :establish_connection, to: ActiveRecord::Base def initialize(configuration) - @configuration = configuration.with_indifferent_access + @configuration = configuration end def create establish_master_connection - connection.create_database @configuration['database'] + connection.create_database @configuration.database rescue ActiveRecord::StatementInvalid => e if e.cause.to_s.include?('already exists') raise ActiveRecord::DatabaseAlreadyExists @@ -21,7 +21,7 @@ def create def drop establish_master_connection - connection.drop_database @configuration['database'] + connection.drop_database @configuration.database end def purge @@ -31,12 +31,21 @@ def purge end def structure_dump(*args) - tables = connection.execute("SHOW TABLES FROM #{@configuration['database']}")['data'].flatten + establish_master_connection + + # get all tables + tables = connection.execute("SHOW TABLES FROM #{@configuration.database} WHERE name NOT LIKE '.inner_id.%'")['data'].flatten.map do |table| + next if %w[schema_migrations ar_internal_metadata].include?(table) + connection.show_create_table(table).gsub("#{@configuration.database}.", '') + end.compact + + # sort view to last + tables.sort_by! {|table| table.match(/^CREATE\s+(MATERIALIZED\s+)?VIEW/) ? 1 : 0} + # put to file File.open(args.first, 'w:utf-8') do |file| tables.each do |table| - next if table.match(/\.inner/) - file.puts connection.execute("SHOW CREATE TABLE #{table}")['data'].try(:first).try(:first).gsub("#{@configuration['database']}.", '') + ";\n\n" + file.puts table + ";\n\n" end end end diff --git a/lib/core_extensions/active_record/internal_metadata.rb b/lib/core_extensions/active_record/internal_metadata.rb index 85b28e24..5f59dbfb 100644 --- a/lib/core_extensions/active_record/internal_metadata.rb +++ b/lib/core_extensions/active_record/internal_metadata.rb @@ -1,55 +1,53 @@ module CoreExtensions module ActiveRecord module InternalMetadata - module ClassMethods - - def create_table - return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) - return if table_exists? || !enabled? - - key_options = connection.internal_string_options_for_primary_key - table_options = { - id: false, - options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key', - if_not_exists: true - } - full_config = connection.instance_variable_get(:@config) || {} - - if full_config[:distributed_service_tables] - table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)') - - distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}" - else - distributed_suffix = '' - end - - connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t| - t.string :key, **key_options - t.string :value - t.timestamps - end - end - private + def create_table + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + return if table_exists? || !enabled? + + key_options = connection.internal_string_options_for_primary_key + table_options = { + id: false, + options: 'ReplacingMergeTree(created_at) PARTITION BY key ORDER BY key', + if_not_exists: true + } + full_config = connection.instance_variable_get(:@config) || {} - def update_entry(key, new_value) - return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + if full_config[:distributed_service_tables] + table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(created_at)') - create_entry(key, new_value) + distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}" + else + distributed_suffix = '' end - def select_entry(key) - return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t| + t.string :key, **key_options + t.string :value + t.timestamps + end + end - sm = ::Arel::SelectManager.new(arel_table) - sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/ - sm.project(::Arel.star) - sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key))) - sm.order(arel_table[primary_key].asc) - sm.limit = 1 + private - connection.select_one(sm, "#{self.class} Load") - end + def update_entry(key, new_value) + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + + create_entry(key, new_value) + end + + def select_entry(key) + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + + sm = ::Arel::SelectManager.new(arel_table) + sm.final! if connection.table_options(table_name)[:options] =~ /^ReplacingMergeTree/ + sm.project(::Arel.star) + sm.where(arel_table[primary_key].eq(::Arel::Nodes::BindParam.new(key))) + sm.order(arel_table[primary_key].asc) + sm.limit = 1 + + connection.select_one(sm, "#{self.class} Load") end end end diff --git a/core_extensions/active_record/migration/command_recorder.rb b/lib/core_extensions/active_record/migration/command_recorder.rb similarity index 100% rename from core_extensions/active_record/migration/command_recorder.rb rename to lib/core_extensions/active_record/migration/command_recorder.rb diff --git a/lib/core_extensions/active_record/schema_migration.rb b/lib/core_extensions/active_record/schema_migration.rb index 6f36ad97..0da6a7ee 100644 --- a/lib/core_extensions/active_record/schema_migration.rb +++ b/lib/core_extensions/active_record/schema_migration.rb @@ -1,47 +1,45 @@ module CoreExtensions module ActiveRecord module SchemaMigration - module ClassMethods - def create_table - return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + def create_table + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) - return if table_exists? + return if table_exists? - version_options = connection.internal_string_options_for_primary_key - table_options = { - id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true - } - full_config = connection.instance_variable_get(:@config) || {} + version_options = connection.internal_string_options_for_primary_key + table_options = { + id: false, options: 'ReplacingMergeTree(ver) ORDER BY (version)', if_not_exists: true + } + full_config = connection.instance_variable_get(:@config) || {} - if full_config[:distributed_service_tables] - table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)') + if full_config[:distributed_service_tables] + table_options.merge!(with_distributed: table_name, sharding_key: 'cityHash64(version)') - distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}" - else - distributed_suffix = '' - end + distributed_suffix = "_#{full_config[:distributed_service_tables_suffix] || 'distributed'}" + else + distributed_suffix = '' + end - connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t| - t.string :version, **version_options - t.column :active, 'Int8', null: false, default: '1' - t.datetime :ver, null: false, default: -> { 'now()' } - end + connection.create_table(table_name + distributed_suffix.to_s, **table_options) do |t| + t.string :version, **version_options + t.column :active, 'Int8', null: false, default: '1' + t.datetime :ver, null: false, default: -> { 'now()' } end + end - def delete_version(version) - return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + def delete_version(version) + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) - im = ::Arel::InsertManager.new(arel_table) - im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0) - connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version) - end + im = ::Arel::InsertManager.new(arel_table) + im.insert(arel_table[primary_key] => version.to_s, arel_table['active'] => 0) + connection.insert(im, "#{self.class} Create Rollback Version", primary_key, version) + end - def all_versions - return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) + def all_versions + return super unless connection.is_a?(::ActiveRecord::ConnectionAdapters::ClickhouseAdapter) - final.where(active: 1).order(:version).pluck(:version) - end + final.where(active: 1).order(:version).pluck(:version) end end end diff --git a/lib/tasks/clickhouse.rake b/lib/tasks/clickhouse.rake index eb37aea5..4854021d 100644 --- a/lib/tasks/clickhouse.rake +++ b/lib/tasks/clickhouse.rake @@ -15,15 +15,18 @@ namespace :clickhouse do # TODO: deprecated desc 'Load database schema' task load: %i[prepare_internal_metadata_table] do + puts 'Warning: `rake clickhouse:schema:load` is deprecated! Use `rake db:schema:load:clickhouse` instead' simple = ENV['simple'] || ARGV.any? { |a| a.include?('--simple') } ? '_simple' : nil ActiveRecord::Base.establish_connection(:clickhouse) - ActiveRecord::SchemaMigration.drop_table + connection = ActiveRecord::Tasks::DatabaseTasks.migration_connection + connection.schema_migration.drop_table load(Rails.root.join("db/clickhouse_schema#{simple}.rb")) end # TODO: deprecated desc 'Dump database schema' task dump: :environment do |_, args| + puts 'Warning: `rake clickhouse:schema:dump` is deprecated! Use `rake db:schema:dump:clickhouse` instead' simple = ENV['simple'] || args[:simple] || ARGV.any? { |a| a.include?('--simple') } ? '_simple' : nil filename = Rails.root.join("db/clickhouse_schema#{simple}.rb") File.open(filename, 'w:utf-8') do |file| @@ -36,43 +39,38 @@ namespace :clickhouse do namespace :structure do desc 'Load database structure' task load: ['db:check_protected_environments'] do - config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') ClickhouseActiverecord::Tasks.new(config).structure_load(Rails.root.join('db/clickhouse_structure.sql')) end desc 'Dump database structure' task dump: ['db:check_protected_environments'] do - config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') ClickhouseActiverecord::Tasks.new(config).structure_dump(Rails.root.join('db/clickhouse_structure.sql')) end end desc 'Creates the database from DATABASE_URL or config/database.yml' task create: [] do - config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') - ActiveRecord::Tasks::DatabaseTasks.create(config) + puts 'Warning: `rake clickhouse:create` is deprecated! Use `rake db:create:clickhouse` instead' end desc 'Drops the database from DATABASE_URL or config/database.yml' task drop: ['db:check_protected_environments'] do - config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') - ActiveRecord::Tasks::DatabaseTasks.drop(config) + puts 'Warning: `rake clickhouse:drop` is deprecated! Use `rake db:drop:clickhouse` instead' end desc 'Empty the database from DATABASE_URL or config/database.yml' task purge: ['db:check_protected_environments'] do - config = ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') - ActiveRecord::Tasks::DatabaseTasks.purge(config) + puts 'Warning: `rake clickhouse:purge` is deprecated! Use `rake db:reset:clickhouse` instead' end # desc 'Resets your database using your migrations for the current environment' task :reset do - Rake::Task['clickhouse:purge'].execute - Rake::Task['clickhouse:migrate'].execute + puts 'Warning: `rake clickhouse:reset` is deprecated! Use `rake db:reset:clickhouse` instead' end desc 'Migrate the clickhouse database' task migrate: %i[prepare_schema_migration_table prepare_internal_metadata_table] do + puts 'Warning: `rake clickhouse:migrate` is deprecated! Use `rake db:migrate:clickhouse` instead' Rake::Task['db:migrate:clickhouse'].execute if File.exist? "#{Rails.root}/db/clickhouse_schema_simple.rb" Rake::Task['clickhouse:schema:dump'].execute(simple: true) @@ -81,9 +79,14 @@ namespace :clickhouse do desc 'Rollback the clickhouse database' task rollback: %i[prepare_schema_migration_table prepare_internal_metadata_table] do + puts 'Warning: `rake clickhouse:rollback` is deprecated! Use `rake db:rollback:clickhouse` instead' Rake::Task['db:rollback:clickhouse'].execute if File.exist? "#{Rails.root}/db/clickhouse_schema_simple.rb" Rake::Task['clickhouse:schema:dump'].execute(simple: true) end end + + def config + ActiveRecord::Base.configurations.configs_for(env_name: Rails.env, name: 'clickhouse') + end end