Merge pull request #2453 from fluent/cmd-option-conf-encoding

command: Add --conf-encoding option to accept non utf-8 configuration file

lib/fluent/command/fluentd.rb

@@ -155,6 +155,10 @@
   opts[:gem_install_path] = s
 }
 
+op.on('--conf-encoding ENCODING', "specify configuration file encoding") { |s|
+  opts[:conf_encoding] = s
+}
+
 if Fluent.windows?
   require 'windows/library'
   include Windows::Library

lib/fluent/supervisor.rb

@@ -249,7 +249,7 @@ def self.load_config(path, params = {})
       config_fname = File.basename(path)
       config_basedir = File.dirname(path)
       # Assume fluent.conf encoding is UTF-8
-      config_data = File.open(path, "r:utf-8:utf-8") {|f| f.read }
+      config_data = File.open(path, "r:#{params['conf_encoding']}:utf-8") {|f| f.read }
       inline_config = params['inline_config']
       if inline_config == '-'
         config_data << "\n" << STDIN.read
@@ -422,6 +422,7 @@ def self.default_options
         supervise: true,
         standalone_worker: false,
         signame: nil,
+        conf_encoding: 'utf-8'
       }
     end
 
@@ -440,6 +441,7 @@ def initialize(opt)
       @config_path = opt[:config_path]
       @inline_config = opt[:inline_config]
       @use_v1_config = opt[:use_v1_config]
+      @conf_encoding = opt[:conf_encoding]
       @log_path = opt[:log_path]
       @dry_run = opt[:dry_run]
       @show_plugin_config = opt[:show_plugin_config]
@@ -618,6 +620,7 @@ def supervise
       params['chuser'] = @chuser
       params['chgroup'] = @chgroup
       params['use_v1_config'] = @use_v1_config
+      params['conf_encoding'] = @conf_encoding
 
       # system config parameters
       params['workers'] = @workers
@@ -763,7 +766,7 @@ def main_process(&block)
     def read_config
       @config_fname = File.basename(@config_path)
       @config_basedir = File.dirname(@config_path)
-      @config_data = File.open(@config_path, "r:utf-8:utf-8") {|f| f.read }
+      @config_data = File.open(@config_path, "r:#{@conf_encoding}:utf-8") {|f| f.read }
       if @inline_config == '-'
         @config_data << "\n" << STDIN.read
       elsif @inline_config

test/command/test_fluentd.rb

@@ -28,9 +28,9 @@ def process_exist?(pid)
     end
   end
 
-  def create_conf_file(name, content)
+  def create_conf_file(name, content, ext_enc = 'utf-8')
     conf_path = File.join(TMP_DIR, name)
-    File.open(conf_path, 'w') do |file|
+    File.open(conf_path, "w:#{ext_enc}:utf-8") do |file|
       file.write content
     end
     conf_path
@@ -203,6 +203,40 @@ def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
     end
   end
 
+  sub_test_case 'with --conf-encoding' do
+    test 'runs successfully' do
+      conf = <<CONF
+# テスト
+<source>
+  @type dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<match dummy>
+  @type null
+</match>
+CONF
+      conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')
+      assert_log_matches(create_cmdline(conf_path, '--conf-encoding', 'shift_jis'), "fluentd worker is now running", 'worker=0')
+    end
+
+    test 'failed to run by invalid encoding' do
+      conf = <<CONF
+# テスト
+<source>
+  @type dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<match dummy>
+  @type null
+</match>
+CONF
+      conf_path = create_conf_file('shift_jis.conf', conf, 'shift_jis')
+      assert_fluentd_fails_to_start(create_cmdline(conf_path), "invalid byte sequence in UTF-8")
+    end
+  end
+
   sub_test_case 'with system configuration about root directory' do
     setup do
       @root_path = File.join(TMP_DIR, "rootpath")

test/test_supervisor.rb

@@ -274,6 +274,7 @@ def test_load_config
     params['log_path'] = 'test/tmp/supervisor/log'
     params['suppress_repeated_stacktrace'] = true
     params['log_level'] = Fluent::Log::LEVEL_INFO
+    params['conf_encoding'] = 'utf-8'
     load_config_proc =  Proc.new { Fluent::Supervisor.load_config(tmp_dir, params) }
 
     # first call
@@ -340,6 +341,7 @@ def test_load_config_for_daemonize
     params['suppress_repeated_stacktrace'] = true
     params['log_level'] = Fluent::Log::LEVEL_INFO
     params['daemonize'] = './fluentd.pid'
+    params['conf_encoding'] = 'utf-8'
     load_config_proc = Proc.new { Fluent::Supervisor.load_config(tmp_dir, params) }
 
     # first call
@@ -414,6 +416,7 @@ def test_load_config_with_multibyte_string
     params['log_path'] = 'test/tmp/supervisor/log'
     params['suppress_repeated_stacktrace'] = true
     params['log_level'] = Fluent::Log::LEVEL_INFO
+    params['conf_encoding'] = 'utf-8'
     load_config_proc =  Proc.new { Fluent::Supervisor.load_config(tmp_path, params) }
 
     se_config = load_config_proc.call