diff --git a/.env b/.env index 64af3fc48c..0416c4fb95 100644 --- a/.env +++ b/.env @@ -5,7 +5,6 @@ RUBYGEMS_URL=https://rubygems.org NPM_URL=https://registry.npmjs.org SECRET_KEY_BASE=bdb4300d46c9d4f116ce3dbbd54cac6b20802d8be1c2333cf5f6f90b1627799ac5d043e8460744077bc0bd6aacdd5c4bf53f499a68303c6752e7f327b874b96a OPENC3_LOGS_BUCKET=logs -OPENC3_GEMS_BUCKET=gems OPENC3_TOOLS_BUCKET=tools OPENC3_CONFIG_BUCKET=config OPENC3_REDIS_HOSTNAME=openc3-redis diff --git a/openc3-cmd-tlm-api/app/controllers/gems_controller.rb b/openc3-cmd-tlm-api/app/controllers/gems_controller.rb index 0bf0fd3627..f0a26d00e6 100644 --- a/openc3-cmd-tlm-api/app/controllers/gems_controller.rb +++ b/openc3-cmd-tlm-api/app/controllers/gems_controller.rb @@ -34,18 +34,18 @@ def create begin gem_file_path = temp_dir + '/' + file.original_filename FileUtils.cp(file.tempfile.path, gem_file_path) - result = OpenC3::GemModel.put(gem_file_path, scope: params[:scope]) + OpenC3::GemModel.put(gem_file_path, gem_install: true, scope: params[:scope]) OpenC3::Logger.info("Gem created: #{params[:gem]}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION'])) + head :ok + rescue => e + OpenC3::Logger.error("Error installing gem: #{file.original_filename}:#{e.formatted}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION'])) + render :json => { :status => 'error', :message => e.message, 'type' => e.class }, :status => 400 ensure FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir) end - if result - head :ok - else - head :internal_server_error - end else - head :internal_server_error + OpenC3::Logger.error("Error installing gem: Gem file as params[:gem] is required", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION'])) + render :json => { :status => 'error', :message => "Gem file as params[:gem] is required" }, :status => 400 end end @@ -53,15 +53,17 @@ def create def destroy return unless authorization('admin') if params[:id] - result = OpenC3::GemModel.destroy(params[:id]) - OpenC3::Logger.info("Gem destroyed: #{params[:id]}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION'])) - if result + begin + OpenC3::GemModel.destroy(params[:id]) + OpenC3::Logger.info("Gem destroyed: #{params[:id]}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION'])) head :ok - else - head :internal_server_error + rescue => e + OpenC3::Logger.error("Error destroying gem: #{params[:id]}:#{e.formatted}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION'])) + render :json => { :status => 'error', :message => e.message, 'type' => e.class }, :status => 400 end else - head :internal_server_error + OpenC3::Logger.error("Error destroying gem: Gem name as params[:id] is required", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION'])) + render :json => { :status => 'error', :message => "Gem name as params[:id] is required" }, :status => 400 end end end diff --git a/openc3-cmd-tlm-api/spec/spec_helper.rb b/openc3-cmd-tlm-api/spec/spec_helper.rb index 6b04d516bb..ae7c694b02 100644 --- a/openc3-cmd-tlm-api/spec/spec_helper.rb +++ b/openc3-cmd-tlm-api/spec/spec_helper.rb @@ -60,7 +60,6 @@ # Disable Redis and Fluentd in the Logger ENV['OPENC3_NO_STORE'] = 'true' ENV['OPENC3_LOGS_BUCKET'] = 'logs' -ENV['OPENC3_GEMS_BUCKET'] = 'gems' ENV['OPENC3_TOOLS_BUCKET'] = 'tools' ENV['OPENC3_CONFIG_BUCKET'] = 'config' ENV['OPENC3_REDIS_HOSTNAME'] = '127.0.0.1' diff --git a/openc3-script-runner-api/spec/spec_helper.rb b/openc3-script-runner-api/spec/spec_helper.rb index 12803d688c..c238a2faef 100644 --- a/openc3-script-runner-api/spec/spec_helper.rb +++ b/openc3-script-runner-api/spec/spec_helper.rb @@ -58,7 +58,6 @@ # Disable Redis and Fluentd in the Logger ENV['OPENC3_NO_STORE'] = 'true' ENV['OPENC3_LOGS_BUCKET'] = 'logs' -ENV['OPENC3_GEMS_BUCKET'] = 'gems' ENV['OPENC3_TOOLS_BUCKET'] = 'tools' ENV['OPENC3_CONFIG_BUCKET'] = 'config' ENV['OPENC3_REDIS_HOSTNAME'] = '127.0.0.1' diff --git a/openc3/bin/openc3cli b/openc3/bin/openc3cli index 2e0604758e..0ff6cc395d 100755 --- a/openc3/bin/openc3cli +++ b/openc3/bin/openc3cli @@ -65,7 +65,7 @@ def print_usage puts " #{MIGRATE_PARSER}" puts " cli bridge CONFIG_FILENAME # Run OpenC3 host bridge" puts " cli bridgesetup CONFIG_FILENAME # Create a default config file" - puts " cli geminstall GEMFILENAME # Install loaded gem to /gems" + puts " cli geminstall GEMFILENAME SCOPE # Install loaded gem to /gems" puts " cli rubysloc # Counts Ruby SLOC recursively. Run with --help for more info." puts " cli xtce_converter # Convert to and from the XTCE format. Run with --help for more info." puts " cli cstol_converter # Converts CSTOL files (.prc) to OpenC3. Run with --help for more info." @@ -448,8 +448,8 @@ def unload_plugin(plugin_name, scope:) end end -def gem_install(gem_filename) - OpenC3::GemModel.install(gem_filename) +def gem_install(gem_filename, scope:) + OpenC3::GemModel.install(gem_filename, scope: scope) end def get_redis_keys @@ -497,7 +497,7 @@ if __FILE__ == $0 unload_plugin(ARGV[1], scope: ARGV[2]) when 'geminstall' - gem_install(ARGV[1]) + gem_install(ARGV[1], scope: ARGV[2]) when 'generate' generate(ARGV[1..-1]) @@ -588,7 +588,6 @@ if __FILE__ == $0 client = OpenC3::Bucket.getClient() client.create(ENV['OPENC3_CONFIG_BUCKET']) client.create(ENV['OPENC3_LOGS_BUCKET']) - client.create(ENV['OPENC3_GEMS_BUCKET']) client.create(ENV['OPENC3_TOOLS_BUCKET']) client.ensure_public(ENV['OPENC3_TOOLS_BUCKET']) diff --git a/openc3/lib/openc3/models/gem_model.rb b/openc3/lib/openc3/models/gem_model.rb index d80226639f..e02e9e2bac 100644 --- a/openc3/lib/openc3/models/gem_model.rb +++ b/openc3/lib/openc3/models/gem_model.rb @@ -17,15 +17,15 @@ # All changes Copyright 2022, OpenC3, Inc. # All Rights Reserved +require 'fileutils' require 'open-uri' require 'nokogiri' require 'httpclient' require 'rubygems' require 'rubygems/uninstaller' require 'tempfile' -require 'openc3/utilities/bucket' require 'openc3/utilities/process_manager' -require 'openc3/api/api' +require "pathname" module OpenC3 # This class acts like a Model but doesn't inherit from Model because it doesn't @@ -33,34 +33,25 @@ module OpenC3 # and destroy to allow interaction with gem files from the PluginModel and # the GemsController. class GemModel - extend Api - def self.names - bucket = Bucket.getClient() - gems = [] - bucket.list_objects(bucket: ENV['OPENC3_GEMS_BUCKET']).each do |object| - gems << object.key - end - gems + result = Pathname.new("#{ENV['GEM_HOME']}/gems").children.select { |c| c.directory? }.collect { |p| File.basename(p) + '.gem' } + return result.sort end - def self.get(dir, name) - bucket = Bucket.getClient() - path = File.join(dir, name) - bucket.get_object(bucket: ENV['OPENC3_GEMS_BUCKET'], key: name, path: path) - return path + def self.get(name) + path = "#{ENV['GEM_HOME']}/cache/#{name}" + return path if File.exist?(path) + raise "Gem #{name} not found" end def self.put(gem_file_path, gem_install: true, scope:) - bucket = Bucket.getClient() if File.file?(gem_file_path) gem_filename = File.basename(gem_file_path) - Logger.info "Installing gem: #{gem_filename}" - File.open(gem_file_path, 'rb') do |file| - bucket.put_object(bucket: ENV['OPENC3_GEMS_BUCKET'], key: gem_filename, body: file) - end + FileUtils.mkdir_p("#{ENV['GEM_HOME']}/cache") unless Dir.exist?("#{ENV['GEM_HOME']}/cache") + FileUtils.cp(gem_file_path, "#{ENV['GEM_HOME']}/cache/#{File.basename(gem_file_path)}") if gem_install - result = OpenC3::ProcessManager.instance.spawn(["ruby", "/openc3/bin/openc3cli", "geminstall", gem_filename], "gem_install", gem_filename, Time.now + 3600.0, scope: scope) + Logger.info "Installing gem: #{gem_filename}" + result = OpenC3::ProcessManager.instance.spawn(["ruby", "/openc3/bin/openc3cli", "geminstall", gem_filename, scope], "gem_install", gem_filename, Time.now + 3600.0, scope: scope) return result end else @@ -72,12 +63,11 @@ def self.put(gem_file_path, gem_install: true, scope:) end def self.install(name_or_path, scope:) - temp_dir = Dir.mktmpdir begin if File.exist?(name_or_path) gem_file_path = name_or_path else - gem_file_path = get(temp_dir, name_or_path) + gem_file_path = get(name_or_path) end begin rubygems_url = get_setting('rubygems_url', scope: scope) @@ -90,30 +80,33 @@ def self.install(name_or_path, scope:) Gem.done_installing_hooks.clear Gem.install(gem_file_path, "> 0.pre", :build_args => ['--no-document'], :prerelease => true) rescue => err - message = "Gem file #{gem_file_path} error installing to /gems\n#{err.formatted}" + message = "Gem file #{gem_file_path} error installing to #{ENV['GEM_HOME']}\n#{err.formatted}" Logger.error message - ensure - FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir) + raise err end end def self.destroy(name) - bucket = Bucket.getClient() - Logger.info "Removing gem: #{name}" - bucket.delete_object(bucket: ENV['OPENC3_GEMS_BUCKET'], key: name) gem_name, version = self.extract_name_and_version(name) - begin - Gem::Uninstaller.new(gem_name, {:version => version, :force => true}).uninstall - rescue => err - message = "Gem file #{name} error uninstalling\n#{err.formatted}" + plugin_gem_names = PluginModel.gem_names + if plugin_gem_names.include?(name) + message = "Gem file #{name} can't be uninstalled because needed by installed plugin" Logger.error message + raise message + else + begin + Gem::Uninstaller.new(gem_name, {:version => version, :force => true}).uninstall + rescue => err + Logger.error "Gem file #{name} error uninstalling\n#{err.formatted}" + raise err + end end end def self.extract_name_and_version(name) split_name = name.split('-') gem_name = split_name[0..-2].join('-') - version = split_name[-1] + version = File.basename(split_name[-1], '.gem') return gem_name, version end end diff --git a/openc3/lib/openc3/models/plugin_model.rb b/openc3/lib/openc3/models/plugin_model.rb index 7c9af871d3..b77c74f6ff 100644 --- a/openc3/lib/openc3/models/plugin_model.rb +++ b/openc3/lib/openc3/models/plugin_model.rb @@ -45,7 +45,6 @@ class PluginModel < Model attr_accessor :plugin_txt_lines attr_accessor :needs_dependencies - # NOTE: The following three class methods are used by the ModelController # and are reimplemented to enable various Model class methods to work def self.get(name:, scope: nil) @@ -72,7 +71,7 @@ def self.install_phase1(gem_file_path, existing_variables: nil, existing_plugin_ # Load gem to internal gem server OpenC3::GemModel.put(gem_file_path, gem_install: false, scope: scope) unless validate_only else - gem_file_path = OpenC3::GemModel.get(temp_dir, gem_name) + gem_file_path = OpenC3::GemModel.get(gem_name) end # Extract gem and process plugin.txt to determine what VARIABLEs need to be filled in @@ -145,7 +144,7 @@ def self.install_phase2(plugin_hash, scope:, gem_file_path: nil, validate_only: # Get the gem from local gem server if it hasn't been passed unless gem_file_path gem_name = plugin_hash['name'].split("__")[0] - gem_file_path = OpenC3::GemModel.get(temp_dir, gem_name) + gem_file_path = OpenC3::GemModel.get(gem_name) end # Actually install the gem now (slow) @@ -280,5 +279,19 @@ def restore OpenC3::PluginModel.install_phase2(plugin_hash, scope: @scope) @destroyed = false end + + # Get list of plugin gem names across all scopes to prevent uninstall of gems from GemModel + def self.gem_names + result = [] + scopes = ScopeModel.names + scopes.each do |scope| + plugin_names = self.names(scope: scope) + plugin_names.each do |plugin_name| + gem_name = plugin_name.split("__")[0] + result << gem_name unless result.include?(gem_name) + end + end + return result.sort + end end end diff --git a/openc3/lib/openc3/utilities/local_mode.rb b/openc3/lib/openc3/utilities/local_mode.rb index 38e52bef2d..aad3f109f7 100644 --- a/openc3/lib/openc3/utilities/local_mode.rb +++ b/openc3/lib/openc3/utilities/local_mode.rb @@ -277,7 +277,7 @@ def self.update_local_plugin_files(full_folder_path, plugin_file_path, plugin_ha temp_dir = Dir.mktmpdir begin unless File.exists?(plugin_file_path) - plugin_file_path = OpenC3::GemModel.get(temp_dir, plugin_file_path) + plugin_file_path = OpenC3::GemModel.get(plugin_file_path) end File.open(File.join(full_folder_path, File.basename(plugin_file_path)), 'wb') do |file| data = File.read(plugin_file_path) diff --git a/openc3/spec/models/gem_model_spec.rb b/openc3/spec/models/gem_model_spec.rb index cbcd4379eb..2cf4a58e6c 100644 --- a/openc3/spec/models/gem_model_spec.rb +++ b/openc3/spec/models/gem_model_spec.rb @@ -20,20 +20,31 @@ require 'spec_helper' require 'tempfile' require 'ostruct' +require 'openc3/models/plugin_model' require 'openc3/models/gem_model' require 'openc3/utilities/aws_bucket' +require 'fileutils' module OpenC3 describe GemModel do before(:each) do + mock_redis() + @orig_gem_home = ENV['GEM_HOME'] + @temp_dir = Dir.mktmpdir + ENV['GEM_HOME'] = @temp_dir @scope = "DEFAULT" - @s3 = instance_double("Aws::S3::Client") - @list_result = OpenStruct.new - @list_result.contents = [OpenStruct.new({ key: 'openc3-test1.gem' }), OpenStruct.new({ key: 'openc3-test2.gem' })] - allow(@s3).to receive(:list_objects_v2).and_return(@list_result) - allow(@s3).to receive(:head_bucket).with(any_args) - allow(@s3).to receive(:create_bucket) - allow(Aws::S3::Client).to receive(:new).and_return(@s3) + @gem_list = ['openc3-test1.gem', 'openc3-test2.gem'] + FileUtils.mkdir_p("#{ENV['GEM_HOME']}/cache") + @gem_list.each do |gem| + FileUtils.mkdir_p("#{ENV['GEM_HOME']}/gems/#{File.basename(gem, '.gem')}") + FileUtils.touch("#{ENV['GEM_HOME']}/cache/#{gem}") + end + end + + after(:each) do + FileUtils.remove_entry(@temp_dir) if @temp_dir and File.exist?(@temp_dir) + @temp_dir = nil + ENV['GEM_HOME'] = @orig_gem_home end describe "self.names" do @@ -43,11 +54,9 @@ module OpenC3 end describe "self.get" do - it "copies the gem to the local filesystem" do - response_path = File.join(Dir.pwd, 'openc3-test1.gem') - expect(@s3).to receive(:get_object).with(bucket: 'gems', key: 'openc3-test1.gem', response_target: response_path) - path = GemModel.get(Dir.pwd, 'openc3-test1.gem') - expect(path).to eql response_path + it "get the gem on the local filesystem" do + path = GemModel.get('openc3-test1.gem') + expect(path).to eql "#{ENV['GEM_HOME']}/cache/openc3-test1.gem" end end @@ -61,7 +70,6 @@ module OpenC3 expect(pm).to receive_message_chain(:instance, :spawn) tf = Tempfile.new("openc3-test3.gem") tf.close - expect(@s3).to receive(:put_object).with(bucket: 'gems', key: File.basename(tf.path), body: anything, cache_control: nil, content_type: nil, metadata: nil) GemModel.put(tf.path, scope: 'DEFAULT') tf.unlink end @@ -69,7 +77,9 @@ module OpenC3 describe "self.destroy" do it "removes the gem from the gem server" do - expect(@s3).to receive(:delete_object).with(bucket: 'gems', key: 'openc3-test1.gem') + uninstaller = instance_double("Gem::Uninstaller").as_null_object + expect(Gem::Uninstaller).to receive(:new).and_return(uninstaller) + expect(uninstaller).to receive(:uninstall) GemModel.destroy("openc3-test1.gem") end end diff --git a/openc3/spec/models/plugin_model_spec.rb b/openc3/spec/models/plugin_model_spec.rb index 51f5aadeba..b3719cc512 100644 --- a/openc3/spec/models/plugin_model_spec.rb +++ b/openc3/spec/models/plugin_model_spec.rb @@ -19,6 +19,7 @@ require 'spec_helper' require 'openc3/models/plugin_model' +require 'openc3/utilities/aws_bucket' module OpenC3 describe PluginModel do @@ -104,6 +105,7 @@ module OpenC3 variables = { "folder" => "THE_FOLDER", "name" => "THE_NAME" } # Just stub the instance deploy method + expect(GemModel).to receive(:install).and_return(nil) expect_any_instance_of(ToolModel).to receive(:deploy).with(anything, variables, validate_only: false).and_return(nil) expect_any_instance_of(TargetModel).to receive(:deploy).with(anything, variables, validate_only: false).and_return(nil) plugin_model = PluginModel.install_phase2({"name" => "name", "variables" => variables, "plugin_txt_lines" => ["TOOL THE_FOLDER THE_NAME", " URL myurl", "TARGET THE_FOLDER THE_NAME"]}, scope: "DEFAULT") diff --git a/openc3/spec/spec_helper.rb b/openc3/spec/spec_helper.rb index eae30f20bd..9b6af3b0bc 100644 --- a/openc3/spec/spec_helper.rb +++ b/openc3/spec/spec_helper.rb @@ -54,7 +54,6 @@ # Disable Redis and Fluentd in the Logger ENV['OPENC3_NO_STORE'] = 'true' ENV['OPENC3_LOGS_BUCKET'] = 'logs' -ENV['OPENC3_GEMS_BUCKET'] = 'gems' ENV['OPENC3_TOOLS_BUCKET'] = 'tools' ENV['OPENC3_CONFIG_BUCKET'] = 'config' # Set some usernames / passwords