Skip to content

Commit

Permalink
Merge pull request #612 from Shopify/pm/lazy-loadable-backend
Browse files Browse the repository at this point in the history
LazyLoadable Backend
  • Loading branch information
radar authored Feb 8, 2022
2 parents 00fc810 + 422959a commit fb7095a
Show file tree
Hide file tree
Showing 8 changed files with 479 additions and 7 deletions.
3 changes: 2 additions & 1 deletion lib/i18n/backend.rb
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,16 @@
module I18n
module Backend
autoload :Base, 'i18n/backend/base'
autoload :InterpolationCompiler, 'i18n/backend/interpolation_compiler'
autoload :Cache, 'i18n/backend/cache'
autoload :CacheFile, 'i18n/backend/cache_file'
autoload :Cascade, 'i18n/backend/cascade'
autoload :Chain, 'i18n/backend/chain'
autoload :Fallbacks, 'i18n/backend/fallbacks'
autoload :Flatten, 'i18n/backend/flatten'
autoload :Gettext, 'i18n/backend/gettext'
autoload :InterpolationCompiler, 'i18n/backend/interpolation_compiler'
autoload :KeyValue, 'i18n/backend/key_value'
autoload :LazyLoadable, 'i18n/backend/lazy_loadable'
autoload :Memoize, 'i18n/backend/memoize'
autoload :Metadata, 'i18n/backend/metadata'
autoload :Pluralization, 'i18n/backend/pluralization'
Expand Down
7 changes: 6 additions & 1 deletion lib/i18n/backend/base.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@ module Base
# for details.
def load_translations(*filenames)
filenames = I18n.load_path if filenames.empty?
filenames.flatten.each { |filename| load_file(filename) }
filenames.flatten.each do |filename|
loaded_translations = load_file(filename)
yield filename, loaded_translations if block_given?
end
end

# This method receives a locale, a data hash and options for storing translations.
Expand Down Expand Up @@ -226,6 +229,8 @@ def load_file(filename)
raise InvalidLocaleData.new(filename, 'expects it to return a hash, but does not')
end
data.each { |locale, d| store_translations(locale, d || {}, skip_symbolize_keys: keys_symbolized) }

data
end

# Loads a plain Ruby translations file. eval'ing the file must yield
Expand Down
184 changes: 184 additions & 0 deletions lib/i18n/backend/lazy_loadable.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
# frozen_string_literal: true

module I18n
module Backend
# Backend that lazy loads translations based on the current locale. This
# implementation avoids loading all translations up front. Instead, it only
# loads the translations that belong to the current locale. This offers a
# performance incentive in local development and test environments for
# applications with many translations for many different locales. It's
# particularly useful when the application only refers to a single locales'
# translations at a time (ex. A Rails workload). The implementation
# identifies which translation files from the load path belong to the
# current locale by pattern matching against their path name.
#
# Specifically, a translation file is considered to belong to a locale if:
# a) the filename is in the I18n load path
# b) the filename ends in a supported extension (ie. .yml, .json, .po, .rb)
# c) the filename starts with the locale identifier
# d) the locale identifier and optional proceeding text is separated by an underscore, ie. "_".
#
# Examples:
# Valid files that will be selected by this backend:
#
# "files/locales/en_translation.yml" (Selected for locale "en")
# "files/locales/fr.po" (Selected for locale "fr")
#
# Invalid files that won't be selected by this backend:
#
# "files/locales/translation-file"
# "files/locales/en-translation.unsupported"
# "files/locales/french/translation.yml"
# "files/locales/fr/translation.yml"
#
# The implementation uses this assumption to defer the loading of
# translation files until the current locale actually requires them.
#
# The backend has two working modes: lazy_load and eager_load.
#
# Note: This backend should only be enabled in test environments!
# When the mode is set to false, the backend behaves exactly like the
# Simple backend, with an additional check that the paths being loaded
# abide by the format. If paths can't be matched to the format, an error is raised.
#
# You can configure lazy loaded backends through the initializer or backends
# accessor:
#
# # In test environments
#
# I18n.backend = I18n::Backend::LazyLoadable.new(lazy_load: true)
#
# # In other environments, such as production and CI
#
# I18n.backend = I18n::Backend::LazyLoadable.new(lazy_load: false) # default
#
class LocaleExtractor
class << self
def locale_from_path(path)
name = File.basename(path, ".*")
locale = name.split("_").first
locale.to_sym unless locale.nil?
end
end
end

class LazyLoadable < Simple
def initialize(lazy_load: false)
@lazy_load = lazy_load
end

# Returns whether the current locale is initialized.
def initialized?
if lazy_load?
initialized_locales[I18n.locale]
else
super
end
end

# Clean up translations and uninitialize all locales.
def reload!
if lazy_load?
@initialized_locales = nil
@translations = nil
else
super
end
end

# Eager loading is not supported in the lazy context.
def eager_load!
if lazy_load?
raise UnsupportedMethod.new(__method__, self.class, "Cannot eager load translations because backend was configured with lazy_load: true.")
else
super
end
end

# Parse the load path and extract all locales.
def available_locales
if lazy_load?
I18n.load_path.map { |path| LocaleExtractor.locale_from_path(path) }
else
super
end
end

def lookup(locale, key, scope = [], options = EMPTY_HASH)
if lazy_load?
I18n.with_locale(locale) do
super
end
else
super
end
end

protected


# Load translations from files that belong to the current locale.
def init_translations
file_errors = if lazy_load?
initialized_locales[I18n.locale] = true
load_translations_and_collect_file_errors(filenames_for_current_locale)
else
@initialized = true
load_translations_and_collect_file_errors(I18n.load_path)
end

raise InvalidFilenames.new(file_errors) unless file_errors.empty?
end

def initialized_locales
@initialized_locales ||= Hash.new(false)
end

private

def lazy_load?
@lazy_load
end

class FilenameIncorrect < StandardError
def initialize(file, expected_locale, unexpected_locales)
super "#{file} can only load translations for \"#{expected_locale}\". Found translations for: #{unexpected_locales}."
end
end

# Loads each file supplied and asserts that the file only loads
# translations as expected by the name. The method returns a list of
# errors corresponding to offending files.
def load_translations_and_collect_file_errors(files)
errors = []

load_translations(files) do |file, loaded_translations|
assert_file_named_correctly!(file, loaded_translations)
rescue FilenameIncorrect => e
errors << e
end

errors
end

# Select all files from I18n load path that belong to current locale.
# These files must start with the locale identifier (ie. "en", "pt-BR"),
# followed by an "_" demarcation to separate proceeding text.
def filenames_for_current_locale
I18n.load_path.flatten.select do |path|
LocaleExtractor.locale_from_path(path) == I18n.locale
end
end

# Checks if a filename is named in correspondence to the translations it loaded.
# The locale extracted from the path must be the single locale loaded in the translations.
def assert_file_named_correctly!(file, translations)
loaded_locales = translations.keys.map(&:to_sym)
expected_locale = LocaleExtractor.locale_from_path(file)
unexpected_locales = loaded_locales.reject { |locale| locale == expected_locale }

raise FilenameIncorrect.new(file, expected_locale, unexpected_locales) unless unexpected_locales.empty?
end
end
end
end
34 changes: 34 additions & 0 deletions lib/i18n/exceptions.rb
Original file line number Diff line number Diff line change
Expand Up @@ -110,4 +110,38 @@ def initialize(type, filename)
super "can not load translations from #{filename}, the file type #{type} is not known"
end
end

class UnsupportedMethod < ArgumentError
attr_reader :method, :backend_klass, :msg
def initialize(method, backend_klass, msg)
@method = method
@backend_klass = backend_klass
@msg = msg
super "#{backend_klass} does not support the ##{method} method. #{msg}"
end
end

class InvalidFilenames < ArgumentError
NUMBER_OF_ERRORS_SHOWN = 20
def initialize(file_errors)
super <<~MSG
Found #{file_errors.count} error(s).
The first #{[file_errors.count, NUMBER_OF_ERRORS_SHOWN].min} error(s):
#{file_errors.map(&:message).first(NUMBER_OF_ERRORS_SHOWN).join("\n")}
To use the LazyLoadable backend:
1. Filenames must start with the locale.
2. An underscore must separate the locale with any optional text that follows.
3. The file must only contain translation data for the single locale.
Example:
"/config/locales/fr.yml" which contains:
```yml
fr:
dog:
chien
```
MSG
end
end
end
8 changes: 3 additions & 5 deletions lib/i18n/tests/basics.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,11 @@ def teardown
I18n.available_locales = nil
end

test "available_locales returns the locales stored to the backend by default" do
test "available_locales returns the available_locales produced by the backend, by default" do
I18n.backend.store_translations('de', :foo => 'bar')
I18n.backend.store_translations('en', :foo => 'foo')

assert I18n.available_locales.include?(:de)
assert I18n.available_locales.include?(:en)
assert_equal I18n.available_locales, I18n.backend.available_locales
end

test "available_locales can be set to something else independently from the actual locale data" do
Expand All @@ -24,8 +23,7 @@ def teardown
assert_equal [:foo, :bar], I18n.available_locales

I18n.available_locales = nil
assert I18n.available_locales.include?(:de)
assert I18n.available_locales.include?(:en)
assert_equal I18n.available_locales, I18n.backend.available_locales
end

test "available_locales memoizes when set explicitely" do
Expand Down
24 changes: 24 additions & 0 deletions test/api/lazy_loadable_test.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
require 'test_helper'

class I18nLazyLoadableBackendApiTest < I18n::TestCase
def setup
I18n.backend = I18n::Backend::LazyLoadable.new
super
end

include I18n::Tests::Basics
include I18n::Tests::Defaults
include I18n::Tests::Interpolation
include I18n::Tests::Link
include I18n::Tests::Lookup
include I18n::Tests::Pluralization
include I18n::Tests::Procs
include I18n::Tests::Localization::Date
include I18n::Tests::Localization::DateTime
include I18n::Tests::Localization::Time
include I18n::Tests::Localization::Procs

test "make sure we use the LazyLoadable backend" do
assert_equal I18n::Backend::LazyLoadable, I18n.backend.class
end
end
Loading

0 comments on commit fb7095a

Please sign in to comment.