Added PostgreSQL support

The database configuration has been extended to also support PostgreSQL
as the database. This is relevant if you are using CoreOS Clair as the
security scanner and you don't want to have multiple databases.

In theory with this implementation we could have support for other RDBMs
as well, but that would mean to add gems for each support and call
`bundle`. To avoid these kinds of scenarios, we stick with just two options:
with MariaDB (the currently supported option) and PostgreSQL.

Signed-off-by: Miquel Sabaté Solà <msabate@suse.com>

.travis.yml

@@ -2,7 +2,6 @@ language: ruby
 sudo: required
 dist: trusty
 rvm:
-
   # Stable versions
   - 2.3.4
   - 2.4.1
@@ -40,6 +39,7 @@ before_install:
 
 before_script:
   - mysql -e 'create database portus_test;'
+  - psql -c 'create database portus_test' -U postgres
 
 script:
   # Compile assets
@@ -58,13 +58,20 @@ script:
 
 env:
   global:
-  # OSC_CREDENTIALS are kept secret
-  # In order to update them, run:
-  #   travis encrypt OSC_CREDENTIALS=user:password
-  # more info at: http://docs.travis-ci.com/user/environment-variables/#Secure-Variables
-  - OBS_REPO=Virtualization:containers:Portus
-  - OBS_BRANCH=master
-  - secure: "SgH8Wvm6n71GTZqTk2Ww/vLg3KK1QhgBMRSs+85TFemJvUDEWLKtANGI2szlqJOfOEWqHZ4rdzgMamtYfGmOZvjUBebaIXlYMcPyKSmcQZX/80R1/v7KFyhP2vqrb+ok/rK6lLXX06jDhohUS04vnx4i1EtoxH62dKImdlZsjzI="
+    # OSC_CREDENTIALS are kept secret
+    # In order to update them, run:
+    #   travis encrypt OSC_CREDENTIALS=user:password
+    # more info at: http://docs.travis-ci.com/user/environment-variables/#Secure-Variables
+    - OBS_REPO=Virtualization:containers:Portus
+    - OBS_BRANCH=master
+    - secure: "SgH8Wvm6n71GTZqTk2Ww/vLg3KK1QhgBMRSs+85TFemJvUDEWLKtANGI2szlqJOfOEWqHZ4rdzgMamtYfGmOZvjUBebaIXlYMcPyKSmcQZX/80R1/v7KFyhP2vqrb+ok/rK6lLXX06jDhohUS04vnx4i1EtoxH62dKImdlZsjzI="
+
+    # DB options for both adapters and Travis.
+    - PORTUS_DB_USERNAME=travis
+
+  matrix:
+    - PORTUS_DB_ADAPTER=mysql2
+    - PORTUS_DB_ADAPTER=postgresql
 
 after_success:
   - packaging/suse/package_and_push_to_obs.sh
@@ -73,3 +80,4 @@ addons:
   code_climate:
     repo_token: 18a0cf6c35e0c801678f12f444051c33e0390ce0efa91ec06a2aa5068b10c19e
   mariadb: '10.2'
+  postgresql: '9.6'

Dockerfile

@@ -16,7 +16,7 @@ COPY Gemfile* ./
 #      installed with the devel_basis pattern, and finally we zypper clean -a.
 RUN zypper ref && \
     zypper -n in --no-recommends ruby2.3-devel ruby2.3-rubygem-bundler \
-           libxml2-devel nodejs libmysqlclient-devel libxslt1 && \
+           libxml2-devel nodejs libmysqlclient-devel postgresql-devel libxslt1 && \
     zypper -n in --no-recommends -t pattern devel_basis && \
     bundle install --retry=3 && \
     zypper -n rm wicked wicked-service autoconf automake \

Gemfile

@@ -12,7 +12,6 @@ gem "devise"
 gem "gravatar_image_tag"
 gem "public_activity"
 gem "active_record_union"
-gem "mysql2", "= 0.4.7"
 gem "search_cop"
 gem "kaminari"
 gem "crono"
@@ -33,6 +32,10 @@ gem "omniauth-gitlab"
 
 gem "rack-cors", "~> 1.0.1"
 
+# Supported DBs
+gem "mysql2", "= 0.4.7", group: :mysql
+gem "pg", "~> 0.20.0", group: :postgres
+
 # Pinning these specific versions because that's what we have on OBS.
 gem "ethon", "~> 0.9.0"
 gem "typhoeus", "~> 1.0.2"

Gemfile.lock

@@ -258,6 +258,7 @@ GEM
     paint (1.0.0)
     parser (2.4.0.0)
       ast (~> 2.2)
+    pg (0.20.0)
     poltergeist (1.15.0)
       capybara (~> 2.1)
       cliver (~> 0.3.1)
@@ -497,6 +498,7 @@ DEPENDENCIES
   omniauth-gitlab
   omniauth-google-oauth2
   omniauth-openid
+  pg (~> 0.20.0)
   poltergeist (~> 1.15.0)
   pry-rails
   public_activity

app/controllers/admin/activities_controller.rb

@@ -6,10 +6,10 @@ class Admin::ActivitiesController < Admin::BaseController
   def index
     respond_to do |format|
       format.html do
-        @activities = PublicActivity::Activity.order("created_at DESC").page(params[:page])
+        @activities = PublicActivity::Activity.order(created_at: :desc).page(params[:page])
       end
       format.csv do
-        @activities = PublicActivity::Activity.order("created_at DESC")
+        @activities = PublicActivity::Activity.order(created_at: :desc)
         headers["Content-Disposition"] = 'attachment; filename="activities.csv"'
         headers["Content-Type"] = "text/csv"
       end

app/controllers/admin/dashboard_controller.rb

@@ -1,7 +1,7 @@
 class Admin::DashboardController < Admin::BaseController
   def index
     @recent_activities = PublicActivity::Activity
-                         .order("created_at DESC")
+                         .order(created_at: :desc)
                          .limit(20)
     @portus_exists = User.where(username: "portus").any?
   end

app/controllers/admin/namespaces_controller.rb

@@ -3,7 +3,7 @@ def index
     @special_namespaces = Namespace.where(global: true)
     @namespaces = Namespace.not_portus
                            .where(global: false)
-                           .order("created_at ASC")
+                           .order(created_at: :asc)
                            .page(params[:page])
   end
 end

app/models/namespace.rb

@@ -13,7 +13,6 @@
 #
 # Indexes
 #
-#  fulltext_index_namespaces_on_name         (name)
 #  index_namespaces_on_name_and_registry_id  (name,registry_id) UNIQUE
 #  index_namespaces_on_registry_id           (registry_id)
 #  index_namespaces_on_team_id               (team_id)

app/models/repository.rb

@@ -11,7 +11,6 @@
 #
 # Indexes
 #
-#  fulltext_index_repositories_on_name          (name)
 #  index_repositories_on_name_and_namespace_id  (name,namespace_id) UNIQUE
 #  index_repositories_on_namespace_id           (namespace_id)
 #
@@ -33,12 +32,6 @@ class Repository < ActiveRecord::Base
   search_scope :search do
     attributes :name
     attributes namespace_name: "namespace.name"
-
-    # TODO: (mssola): we are experiencing some issues with MariaDB's fulltext
-    # support. Because of that, the following two options have been disabled
-    # until we find a solution for it.
-    # options :name, type: :fulltext
-    # options :namespace_name, type: :fulltext
   end
 
   # Returns the full name for this repository. What this means is that it
@@ -64,8 +57,8 @@ def starred_by?(user)
   # digest.
   def groupped_tags
     tags.group_by(&:digest).values.sort do |x, y|
-      y.first.updated_at <=> x.first.updated_at
-    end
+      y.first.updated_at.to_f <=> x.first.updated_at.to_f
+    end.map(&:sort)
   end
 
   # Updates the activities related to this repository and adds a new activity

config/database.yml

@@ -1,38 +1,37 @@
-default: &default
-  adapter: mysql2
-  encoding: utf8
-<% if ENV['COMPOSE'] %>
-  host: <%= ENV['PORTUS_DB_HOST'] %>
-  username: root
-  password: portus
-<% end %>
+# You could theoretically use anything for the database with this setup, but we
+# have decided on two choices: MariaDB (default) and PostgreSQL. In order to
+# support other RDBMS we would need to add them in the Gemfile and call
+# `bundle`, which is a burden in containerized scenarios.
+#
+# You can pick the adapter with the `PORTUS_DB_ADAPTER` environment variable
+# (defaults to 'mysql2'. If you want to use PostgreSQL you need to set it to
+# `postgresql`).
+#
+# For more documentation check:
+#  - The `examples/development/postgresql` directory. Read the `README.md` file
+#    for more information.
+#  - Our mailing list: https://groups.google.com/forum/#!forum/portus-dev.
 
-development:
-  <<: *default
-  database: portus_development
+default: &default
+  adapter:  <%= ENV["PORTUS_DB_ADAPTER"]  || "mysql2"    %>
 
-staging:
-  <<: *default
-  database: portus_staging
+  host:     <%= ENV['PORTUS_DB_HOST']     || "localhost" %>
+  username: <%= ENV['PORTUS_DB_USERNAME'] || "root"      %>
+  password: <%= ENV['PORTUS_DB_PASSWORD'] || ""          %>
 
-# Warning: The database defined as "test" will be erased and
-# re-generated from your development database when you run "rake".
-# Do not set this db to the same as development or production.
-test:
-  <<: *default
-  database: portus_test
+  encoding: utf8
 
-production:
-  <<: *default
-  <% if ENV["PORTUS_PRODUCTION_HOST"] %>
-  host:     <%= ENV["PORTUS_PRODUCTION_HOST"] %>
-  <% end %>
-  <% if ENV["PORTUS_PRODUCTION_USERNAME"] %>
-  username: <%= ENV["PORTUS_PRODUCTION_USERNAME"] %>
+  # Optional stuff.
+  <% if ENV["PORTUS_DB_PORT"] %>
+  port: <%= ENV['PORTUS_DB_PORT'] %>
   <% end %>
-  <% if ENV["PORTUS_PRODUCTION_PASSWORD"] %>
-  password: <%= ENV["PORTUS_PRODUCTION_PASSWORD"] %>
+  <% if ENV["PORTUS_DB_POOL"] %>
+  pool: <%= ENV['PORTUS_DB_POOL'] %>
   <% end %>
-  <% if ENV["PORTUS_PRODUCTION_DATABASE"] %>
-  database: <%= ENV["PORTUS_PRODUCTION_DATABASE"] %>
+  <% if ENV["PORTUS_DB_TIMEOUT"] %>
+  timeout: <%= ENV['PORTUS_DB_TIMEOUT'] %>
   <% end %>
+
+<%= Rails.env.downcase %>:
+  <<: *default
+  database: <%= ENV["PORTUS_DB_DATABASE"] || "portus_#{Rails.env.downcase}" %>

db/migrate/20150521145620_add_fulltext_indexes_on_repositories_namespaces.rb

@@ -1,6 +1,12 @@
 class AddFulltextIndexesOnRepositoriesNamespaces < ActiveRecord::Migration
   def change
-    add_index :namespaces, :name, type: :fulltext, name: 'fulltext_index_namespaces_on_name'
-    add_index :repositories, :name, type: :fulltext, name: 'fulltext_index_repositories_on_name'
+    # This `if` statement has been added after this migration was
+    # created. Modifying migrations is generally a *bad* idea but this will only
+    # apply to PostgreSQL deployments that start from scratch, which haven't
+    # been supported until now.
+    if ENV["PORTUS_DB_ADAPTER"].blank? || ENV["PORTUS_DB_ADAPTER"] == "mysql"
+      add_index :namespaces, :name, type: :fulltext, name: 'fulltext_index_namespaces_on_name'
+      add_index :repositories, :name, type: :fulltext, name: 'fulltext_index_repositories_on_name'
+    end
   end
 end

db/migrate/20171011150408_remove_fulltext_index_repositories_on_name_index.rb

@@ -0,0 +1,10 @@
+class RemoveFulltextIndexRepositoriesOnNameIndex < ActiveRecord::Migration
+  def change
+    remove_index_if_exists :repositories, "fulltext_index_repositories_on_name"
+    remove_index_if_exists :namespaces, "fulltext_index_namespaces_on_name"
+  end
+
+  def remove_index_if_exists(table, name)
+    remove_index table, name: name if index_exists?(table, :name, name: name)
+  end
+end

db/schema.rb

@@ -74,7 +74,6 @@
   end
 
   add_index "namespaces", ["name", "registry_id"], name: "index_namespaces_on_name_and_registry_id", unique: true, using: :btree
-  add_index "namespaces", ["name"], name: "fulltext_index_namespaces_on_name", type: :fulltext
   add_index "namespaces", ["registry_id"], name: "index_namespaces_on_registry_id", using: :btree
   add_index "namespaces", ["team_id"], name: "index_namespaces_on_team_id", using: :btree
 
@@ -107,7 +106,6 @@
   end
 
   add_index "repositories", ["name", "namespace_id"], name: "index_repositories_on_name_and_namespace_id", unique: true, using: :btree
-  add_index "repositories", ["name"], name: "fulltext_index_repositories_on_name", type: :fulltext
   add_index "repositories", ["namespace_id"], name: "index_repositories_on_namespace_id", using: :btree
 
   create_table "stars", force: :cascade do |t|

docker-compose.yml

@@ -11,6 +11,7 @@ services:
 
       - PORTUS_DB_HOST=db
       - PORTUS_DB_PASSWORD=portus
+      - PORTUS_DB_POOL=5
 
       - RAILS_SERVE_STATIC_FILES=true
     ports:
@@ -30,6 +31,7 @@ services:
 
       - PORTUS_DB_HOST=db
       - PORTUS_DB_PASSWORD=portus
+      - PORTUS_DB_POOL=5
     volumes:
       - .:/srv/Portus
     links:

examples/compose/docker-compose.insecure.yml

@@ -11,6 +11,7 @@ services:
       PORTUS_PRODUCTION_HOST: db
       PORTUS_PRODUCTION_DATABASE: portus_production
       PORTUS_PRODUCTION_PASSWORD: ${DATABASE_PASSWORD}
+      PORTUS_DB_POOL=5
 
       # Secrets. It can possibly be handled better with Swarm's secrets.
       PORTUS_SECRET_KEY_BASE: ${SECRET_KEY_BASE}
@@ -40,6 +41,7 @@ services:
       PORTUS_PRODUCTION_HOST: db
       PORTUS_PRODUCTION_DATABASE: portus_production
       PORTUS_PRODUCTION_PASSWORD: ${DATABASE_PASSWORD}
+      PORTUS_DB_POOL=5
 
       # Secrets. It can possibly be handled better with Swarm's secrets.
       PORTUS_SECRET_KEY_BASE: ${SECRET_KEY_BASE}

examples/compose/docker-compose.yml

@@ -11,6 +11,7 @@ services:
       PORTUS_PRODUCTION_HOST: db
       PORTUS_PRODUCTION_DATABASE: portus_production
       PORTUS_PRODUCTION_PASSWORD: ${DATABASE_PASSWORD}
+      PORTUS_DB_POOL=5
 
       # Secrets. It can possibly be handled better with Swarm's secrets.
       PORTUS_SECRET_KEY_BASE: ${SECRET_KEY_BASE}
@@ -42,6 +43,7 @@ services:
       PORTUS_PRODUCTION_HOST: db
       PORTUS_PRODUCTION_DATABASE: portus_production
       PORTUS_PRODUCTION_PASSWORD: ${DATABASE_PASSWORD}
+      PORTUS_DB_POOL=5
 
       # Secrets. It can possibly be handled better with Swarm's secrets.
       PORTUS_SECRET_KEY_BASE: ${SECRET_KEY_BASE}

examples/development/postgresql/README.md

@@ -0,0 +1,23 @@
+# About this example
+
+This example runs Portus using PostgreSQL instead of MySQL. This is achieved
+through environment variables. You have to set `PORTUS_DB_ADAPTER` to
+`postgresql` so this RDBMS is picked instead of the default MYSQL adapter.
+
+## How to run this example
+
+This example is similar to the `docker-compose.yml` file from the root of the
+project, but using PostgreSQL. If you want to use it, perform the following
+commands (from the root of the project):
+
+```
+$ cp examples/postgresql/docker-compose.yml docker-compose.postgres.yml
+$ docker-compose -f docker-compose.postgres.yml up
+```
+
+## Tips for production
+
+If you want to run PostgreSQL and Portus in production, having to call `bundle`
+when bringing up the containers is a bad idea. Instead, create a new Docker
+image that derives from the [official Portus image](https://github.com/openSUSE/docker-containers/tree/master/derived_images/portus) and install the `pg` gem
+there.