From 19cfe9fcc4ba815bc529d7789037f5bb7d32dc8b Mon Sep 17 00:00:00 2001 From: leonardobrito Date: Wed, 4 Oct 2023 21:58:42 -0300 Subject: [PATCH 1/2] Add elasticsearch setup --- Gemfile | 7 ++-- Gemfile.lock | 40 +++++++++++++++++++++++ app/chewy/users_index.rb | 15 +++++++++ app/controllers/hello_world_controller.rb | 12 +++++++ app/models/user.rb | 2 ++ config/chewy.yml | 7 ++++ config/initializers/chewy.rb | 18 ++++++++++ config/routes.rb | 1 + docker-compose.yml | 21 ++++++++++++ spec/requests/hello_world_spec.rb | 19 +++++++++++ spec/support/chewy.rb | 7 ++++ 11 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 app/chewy/users_index.rb create mode 100644 config/chewy.yml create mode 100644 config/initializers/chewy.rb create mode 100644 spec/support/chewy.rb diff --git a/Gemfile b/Gemfile index 76c50fd..803178e 100644 --- a/Gemfile +++ b/Gemfile @@ -62,6 +62,9 @@ gem "rswag" # Simple, efficient background processing for Ruby [https://github.com/sidekiq/sidekiq] gem "sidekiq" +# Elasticsearch with chewy [https://github.com/toptal/chewy] +gem "chewy" + group :development, :test do gem "bullet" @@ -80,13 +83,13 @@ group :development do gem "web-console" gem "brakeman" + gem "pry", "~> 0.14.2" + gem "reek" # Add speed badges [https://github.com/MiniProfiler/rack-mini-profiler] # gem "rack-mini-profiler" # Speed up commands on slow machines / big apps [https://github.com/rails/spring] gem "spring" - gem "pry", "~> 0.14.2" - gem "reek" end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index 8ba66e6..a4727e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -101,6 +101,10 @@ GEM rack-test (>= 0.6.3) regexp_parser (>= 1.5, < 3.0) xpath (~> 3.2) + chewy (7.3.4) + activesupport (>= 5.2) + elasticsearch (>= 7.12.0, < 7.14.0) + elasticsearch-dsl coderay (1.1.3) concurrent-ruby (1.2.2) connection_pool (2.4.1) @@ -139,12 +143,44 @@ GEM dry-inflector (~> 1.0) dry-logic (~> 1.4) zeitwerk (~> 2.6) + elasticsearch (7.13.3) + elasticsearch-api (= 7.13.3) + elasticsearch-transport (= 7.13.3) + elasticsearch-api (7.13.3) + multi_json + elasticsearch-dsl (0.1.10) + elasticsearch-transport (7.13.3) + faraday (~> 1) + multi_json erubi (1.12.0) factory_bot (6.2.1) activesupport (>= 5.0.0) factory_bot_rails (6.2.0) factory_bot (~> 6.2.0) railties (>= 5.0.0) + faraday (1.10.3) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0) + faraday-multipart (~> 1.0) + faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.0) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) + faraday-retry (~> 1.0) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-multipart (1.0.4) + multipart-post (~> 2) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + faraday-retry (1.0.3) globalid (1.2.1) activesupport (>= 6.1) i18n (1.14.1) @@ -179,6 +215,8 @@ GEM mini_portile2 (2.8.4) minitest (5.20.0) msgpack (1.7.2) + multi_json (1.15.0) + multipart-post (2.3.0) net-imap (0.3.7) date net-protocol @@ -327,6 +365,7 @@ GEM rubocop-capybara (~> 2.17) rubocop-factory_bot (~> 2.22) ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) rubyzip (2.3.2) selenium-webdriver (4.12.0) rexml (~> 3.2, >= 3.2.5) @@ -393,6 +432,7 @@ DEPENDENCIES brakeman bullet capybara + chewy debug devise (~> 4.9) devise-api! diff --git a/app/chewy/users_index.rb b/app/chewy/users_index.rb new file mode 100644 index 0000000..40821fc --- /dev/null +++ b/app/chewy/users_index.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class UsersIndex < Chewy::Index + settings analysis: { + analyzer: { + email: { + tokenizer: "keyword", + filter: ["lowercase"] + } + } + } + + index_scope User + field :email, analyzer: "email" +end diff --git a/app/controllers/hello_world_controller.rb b/app/controllers/hello_world_controller.rb index a3b8bd6..4827dd2 100644 --- a/app/controllers/hello_world_controller.rb +++ b/app/controllers/hello_world_controller.rb @@ -10,4 +10,16 @@ def public_method def private_method render plain: "This method needs authentication" end + + def search + query = search_params[:query] + @users = UsersIndex.query(query_string: { fields: [:email], query: query, default_operator: "and" }) + render json: @users.objects.to_json, status: :ok + end + + private + + def search_params + params.permit(:query, :page, :per) + end end diff --git a/app/models/user.rb b/app/models/user.rb index 206142b..189b10a 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class User < ApplicationRecord + update_index("users") { self } + # Include default devise modules. Others available are: # :confirmable, :lockable, :timeoutable, :trackable and :omniauthable devise :database_authenticatable, :registerable, diff --git a/config/chewy.yml b/config/chewy.yml new file mode 100644 index 0000000..3719715 --- /dev/null +++ b/config/chewy.yml @@ -0,0 +1,7 @@ +# config/chewy.yml +# separate environment configs +test: + host: "<%= ENV['ELASTICSEARCH_HOST'] %>" + prefix: 'test' +development: + host: "<%= ENV['ELASTICSEARCH_HOST'] %>" diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb new file mode 100644 index 0000000..6aaea42 --- /dev/null +++ b/config/initializers/chewy.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +Chewy.strategy(:atomic) +Chewy.request_strategy = :sidekiq +Chewy.logger = Logger.new($stdout) +Chewy.logger.level = Logger::INFO +Chewy.settings = { + strategy_config: { + delayed_sidekiq: { + latency: 3, + margin: 2, + ttl: 60 * 60 * 24, + reindex_wrapper: lambda { |&reindex| + ActiveRecord::Base.connected_to(role: :reading) { reindex.call } + } + } + } +} diff --git a/config/routes.rb b/config/routes.rb index c89b161..6230318 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -14,4 +14,5 @@ get "/public_method", to: "hello_world#public_method" get "/private_method", to: "hello_world#private_method" + get "/search", to: "hello_world#search" end diff --git a/docker-compose.yml b/docker-compose.yml index 23c850a..b2fb0ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,7 @@ x-backend: &backend POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_HOST: database + ELASTICSEARCH_HOST: http://elasticsearch:9200 volumes: - .:/app - backend_data:/usr/local/bundle @@ -45,12 +46,31 @@ services: healthcheck: test: pg_isready -U postgres -h 127.0.0.1 interval: 5s + elasticsearch: + image: elasticsearch:8.10.2 + environment: + - "discovery.type=single-node" + - "cluster.name=elasticsearch-rails" + - "cluster.routing.allocation.disk.threshold_enabled=false" + - "ES_JAVA_OPTS=-Xms2g -Xmx2g" + - ELASTIC_PASSWORD=pwd123 + - xpack.security.enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - elasticsearch_data:/usr/share/elasticsearch/data + ports: + - 9200:9200 web: <<: *backend hostname: web-app-container ports: - "3000:3000" depends_on: + elasticsearch: + condition: service_started sidekiq: condition: service_started sidekiq: @@ -73,3 +93,4 @@ volumes: backend_data: database-data: driver: local + elasticsearch_data: diff --git a/spec/requests/hello_world_spec.rb b/spec/requests/hello_world_spec.rb index 1371615..dbe097b 100644 --- a/spec/requests/hello_world_spec.rb +++ b/spec/requests/hello_world_spec.rb @@ -23,4 +23,23 @@ expect(response.body).to eq("This method needs authentication") end end + + describe "GET /search" do + let!(:user) { create(:user) } + + before do + UsersIndex.import + sign_in user + get "/search", params: { query: user.email } + end + + after do + UsersIndex.delete + end + + it "renders message" do + parsed_body = response.parsed_body + expect(parsed_body).to eq([user.as_json]) + end + end end diff --git a/spec/support/chewy.rb b/spec/support/chewy.rb new file mode 100644 index 0000000..84a755e --- /dev/null +++ b/spec/support/chewy.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +RSpec.configure do |config| + config.before(:suite) do + Chewy.strategy(:bypass) + end +end From 6491b694500ab7d4fdd427039649838a1ac54018 Mon Sep 17 00:00:00 2001 From: leonardobrito Date: Wed, 4 Oct 2023 22:24:33 -0300 Subject: [PATCH 2/2] Add elasticsearch github actions configuration --- .github/workflows/ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d2f213d..6b3f60a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,7 @@ jobs: - scan runs-on: ubuntu-latest env: + ELASTICSEARCH_HOST: http://localhost:9200 DATABASE_URL: postgres://postgres:example@localhost:5432/app_test services: postgres: @@ -79,6 +80,23 @@ jobs: - name: Install Yarn packages run: yarn install --check-files --pure-lockfile + - name: Configure sysctl limits + run: | + sudo swapoff -a + sudo sysctl -w vm.swappiness=1 + sudo sysctl -w fs.file-max=262144 + sudo sysctl -w vm.max_map_count=262144 + + - name: Runs Elasticsearch + uses: elastic/elastic-github-actions/elasticsearch@master + with: + stack-version: 8.10.2 + security-enabled: false + + - name: Elasticsearch is reachable + run: | + curl --verbose --show-error --insecure http://localhost:9200 + - name: Create database structure run: RAILS_ENV=test bundle exec rails db:create db:schema:load