Merge pull request #241 from OpenC3/remove_gems_bucket

Remove GEMS bucket. GemModel directly uses /gems volume instead

.env

@@ -5,7 +5,6 @@ RUBYGEMS_URL=https://rubygems.org
 NPM_URL=https://registry.npmjs.org
 SECRET_KEY_BASE=bdb4300d46c9d4f116ce3dbbd54cac6b20802d8be1c2333cf5f6f90b1627799ac5d043e8460744077bc0bd6aacdd5c4bf53f499a68303c6752e7f327b874b96a
 OPENC3_LOGS_BUCKET=logs
-OPENC3_GEMS_BUCKET=gems
 OPENC3_TOOLS_BUCKET=tools
 OPENC3_CONFIG_BUCKET=config
 OPENC3_REDIS_HOSTNAME=openc3-redis

openc3-cmd-tlm-api/app/controllers/gems_controller.rb

@@ -34,34 +34,36 @@ def create
       begin
         gem_file_path = temp_dir + '/' + file.original_filename
         FileUtils.cp(file.tempfile.path, gem_file_path)
-        result = OpenC3::GemModel.put(gem_file_path, scope: params[:scope])
+        OpenC3::GemModel.put(gem_file_path, gem_install: true, scope: params[:scope])
         OpenC3::Logger.info("Gem created: #{params[:gem]}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
+        head :ok
+      rescue => e
+        OpenC3::Logger.error("Error installing gem: #{file.original_filename}:#{e.formatted}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
+        render :json => { :status => 'error', :message => e.message, 'type' => e.class }, :status => 400
       ensure
         FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir)
       end
-      if result
-        head :ok
-      else
-        head :internal_server_error
-      end
     else
-      head :internal_server_error
+      OpenC3::Logger.error("Error installing gem: Gem file as params[:gem] is required", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
+      render :json => { :status => 'error', :message => "Gem file as params[:gem] is required" }, :status => 400
     end
   end
 
   # Remove a gem
   def destroy
     return unless authorization('admin')
     if params[:id]
-      result = OpenC3::GemModel.destroy(params[:id])
-      OpenC3::Logger.info("Gem destroyed: #{params[:id]}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
-      if result
+      begin
+        OpenC3::GemModel.destroy(params[:id])
+        OpenC3::Logger.info("Gem destroyed: #{params[:id]}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
         head :ok
-      else
-        head :internal_server_error
+      rescue => e
+        OpenC3::Logger.error("Error destroying gem: #{params[:id]}:#{e.formatted}", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
+        render :json => { :status => 'error', :message => e.message, 'type' => e.class }, :status => 400
       end
     else
-      head :internal_server_error
+      OpenC3::Logger.error("Error destroying gem: Gem name as params[:id] is required", scope: params[:scope], user: user_info(request.headers['HTTP_AUTHORIZATION']))
+      render :json => { :status => 'error', :message => "Gem name as params[:id] is required" }, :status => 400
     end
   end
 end

openc3-cmd-tlm-api/spec/spec_helper.rb

@@ -60,7 +60,6 @@
 # Disable Redis and Fluentd in the Logger
 ENV['OPENC3_NO_STORE'] = 'true'
 ENV['OPENC3_LOGS_BUCKET'] = 'logs'
-ENV['OPENC3_GEMS_BUCKET'] = 'gems'
 ENV['OPENC3_TOOLS_BUCKET'] = 'tools'
 ENV['OPENC3_CONFIG_BUCKET'] = 'config'
 ENV['OPENC3_REDIS_HOSTNAME'] = '127.0.0.1'

openc3-script-runner-api/spec/spec_helper.rb

@@ -58,7 +58,6 @@
 # Disable Redis and Fluentd in the Logger
 ENV['OPENC3_NO_STORE'] = 'true'
 ENV['OPENC3_LOGS_BUCKET'] = 'logs'
-ENV['OPENC3_GEMS_BUCKET'] = 'gems'
 ENV['OPENC3_TOOLS_BUCKET'] = 'tools'
 ENV['OPENC3_CONFIG_BUCKET'] = 'config'
 ENV['OPENC3_REDIS_HOSTNAME'] = '127.0.0.1'

openc3/bin/openc3cli

@@ -65,7 +65,7 @@ def print_usage
   puts "  #{MIGRATE_PARSER}"
   puts "  cli bridge CONFIG_FILENAME        # Run OpenC3 host bridge"
   puts "  cli bridgesetup CONFIG_FILENAME   # Create a default config file"
-  puts "  cli geminstall GEMFILENAME        # Install loaded gem to /gems"
+  puts "  cli geminstall GEMFILENAME SCOPE        # Install loaded gem to /gems"
   puts "  cli rubysloc                      # Counts Ruby SLOC recursively. Run with --help for more info."
   puts "  cli xtce_converter                # Convert to and from the XTCE format. Run with --help for more info."
   puts "  cli cstol_converter               # Converts CSTOL files (.prc) to OpenC3. Run with --help for more info."
@@ -448,8 +448,8 @@ def unload_plugin(plugin_name, scope:)
   end
 end
 
-def gem_install(gem_filename)
-  OpenC3::GemModel.install(gem_filename)
+def gem_install(gem_filename, scope:)
+  OpenC3::GemModel.install(gem_filename, scope: scope)
 end
 
 def get_redis_keys
@@ -497,7 +497,7 @@ if __FILE__ == $0
       unload_plugin(ARGV[1], scope: ARGV[2])
 
     when 'geminstall'
-      gem_install(ARGV[1])
+      gem_install(ARGV[1], scope: ARGV[2])
 
     when 'generate'
       generate(ARGV[1..-1])
@@ -588,7 +588,6 @@ if __FILE__ == $0
       client = OpenC3::Bucket.getClient()
       client.create(ENV['OPENC3_CONFIG_BUCKET'])
       client.create(ENV['OPENC3_LOGS_BUCKET'])
-      client.create(ENV['OPENC3_GEMS_BUCKET'])
       client.create(ENV['OPENC3_TOOLS_BUCKET'])
       client.ensure_public(ENV['OPENC3_TOOLS_BUCKET'])
 

openc3/lib/openc3/models/gem_model.rb

@@ -17,50 +17,41 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 
+require 'fileutils'
 require 'open-uri'
 require 'nokogiri'
 require 'httpclient'
 require 'rubygems'
 require 'rubygems/uninstaller'
 require 'tempfile'
-require 'openc3/utilities/bucket'
 require 'openc3/utilities/process_manager'
-require 'openc3/api/api'
+require "pathname"
 
 module OpenC3
   # This class acts like a Model but doesn't inherit from Model because it doesn't
   # actual interact with the Store (Redis). Instead we implement names, get, put
   # and destroy to allow interaction with gem files from the PluginModel and
   # the GemsController.
   class GemModel
-    extend Api
-
     def self.names
-      bucket = Bucket.getClient()
-      gems = []
-      bucket.list_objects(bucket: ENV['OPENC3_GEMS_BUCKET']).each do |object|
-        gems << object.key
-      end
-      gems
+      result = Pathname.new("#{ENV['GEM_HOME']}/gems").children.select { |c| c.directory? }.collect { |p| File.basename(p) + '.gem' }
+      return result.sort
     end
 
-    def self.get(dir, name)
-      bucket = Bucket.getClient()
-      path = File.join(dir, name)
-      bucket.get_object(bucket: ENV['OPENC3_GEMS_BUCKET'], key: name, path: path)
-      return path
+    def self.get(name)
+      path = "#{ENV['GEM_HOME']}/cache/#{name}"
+      return path if File.exist?(path)
+      raise "Gem #{name} not found"
     end
 
     def self.put(gem_file_path, gem_install: true, scope:)
-      bucket = Bucket.getClient()
       if File.file?(gem_file_path)
         gem_filename = File.basename(gem_file_path)
-        Logger.info "Installing gem: #{gem_filename}"
-        File.open(gem_file_path, 'rb') do |file|
-          bucket.put_object(bucket: ENV['OPENC3_GEMS_BUCKET'], key: gem_filename, body: file)
-        end
+        FileUtils.mkdir_p("#{ENV['GEM_HOME']}/cache") unless Dir.exist?("#{ENV['GEM_HOME']}/cache")
+        FileUtils.cp(gem_file_path, "#{ENV['GEM_HOME']}/cache/#{File.basename(gem_file_path)}")
         if gem_install
-          result = OpenC3::ProcessManager.instance.spawn(["ruby", "/openc3/bin/openc3cli", "geminstall", gem_filename], "gem_install", gem_filename, Time.now + 3600.0, scope: scope)
+          Logger.info "Installing gem: #{gem_filename}"
+          result = OpenC3::ProcessManager.instance.spawn(["ruby", "/openc3/bin/openc3cli", "geminstall", gem_filename, scope], "gem_install", gem_filename, Time.now + 3600.0, scope: scope)
           return result
         end
       else
@@ -72,12 +63,11 @@ def self.put(gem_file_path, gem_install: true, scope:)
     end
 
     def self.install(name_or_path, scope:)
-      temp_dir = Dir.mktmpdir
       begin
         if File.exist?(name_or_path)
           gem_file_path = name_or_path
         else
-          gem_file_path = get(temp_dir, name_or_path)
+          gem_file_path = get(name_or_path)
         end
         begin
           rubygems_url = get_setting('rubygems_url', scope: scope)
@@ -90,30 +80,33 @@ def self.install(name_or_path, scope:)
         Gem.done_installing_hooks.clear
         Gem.install(gem_file_path, "> 0.pre", :build_args => ['--no-document'], :prerelease => true)
       rescue => err
-        message = "Gem file #{gem_file_path} error installing to /gems\n#{err.formatted}"
+        message = "Gem file #{gem_file_path} error installing to #{ENV['GEM_HOME']}\n#{err.formatted}"
         Logger.error message
-      ensure
-        FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir)
+        raise err
       end
     end
 
     def self.destroy(name)
-      bucket = Bucket.getClient()
-      Logger.info "Removing gem: #{name}"
-      bucket.delete_object(bucket: ENV['OPENC3_GEMS_BUCKET'], key: name)
       gem_name, version = self.extract_name_and_version(name)
-      begin
-        Gem::Uninstaller.new(gem_name, {:version => version, :force => true}).uninstall
-      rescue => err
-        message = "Gem file #{name} error uninstalling\n#{err.formatted}"
+      plugin_gem_names = PluginModel.gem_names
+      if plugin_gem_names.include?(name)
+        message = "Gem file #{name} can't be uninstalled because needed by installed plugin"
         Logger.error message
+        raise message
+      else
+        begin
+          Gem::Uninstaller.new(gem_name, {:version => version, :force => true}).uninstall
+        rescue => err
+          Logger.error "Gem file #{name} error uninstalling\n#{err.formatted}"
+          raise err
+        end
       end
     end
 
     def self.extract_name_and_version(name)
       split_name = name.split('-')
       gem_name = split_name[0..-2].join('-')
-      version = split_name[-1]
+      version = File.basename(split_name[-1], '.gem')
       return gem_name, version
     end
   end

openc3/lib/openc3/models/plugin_model.rb

@@ -45,7 +45,6 @@ class PluginModel < Model
     attr_accessor :plugin_txt_lines
     attr_accessor :needs_dependencies
 
-
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
     def self.get(name:, scope: nil)
@@ -72,7 +71,7 @@ def self.install_phase1(gem_file_path, existing_variables: nil, existing_plugin_
           # Load gem to internal gem server
           OpenC3::GemModel.put(gem_file_path, gem_install: false, scope: scope) unless validate_only
         else
-          gem_file_path = OpenC3::GemModel.get(temp_dir, gem_name)
+          gem_file_path = OpenC3::GemModel.get(gem_name)
         end
 
         # Extract gem and process plugin.txt to determine what VARIABLEs need to be filled in
@@ -145,7 +144,7 @@ def self.install_phase2(plugin_hash, scope:, gem_file_path: nil, validate_only:
         # Get the gem from local gem server if it hasn't been passed
         unless gem_file_path
           gem_name = plugin_hash['name'].split("__")[0]
-          gem_file_path = OpenC3::GemModel.get(temp_dir, gem_name)
+          gem_file_path = OpenC3::GemModel.get(gem_name)
         end
 
         # Actually install the gem now (slow)
@@ -280,5 +279,19 @@ def restore
       OpenC3::PluginModel.install_phase2(plugin_hash, scope: @scope)
       @destroyed = false
     end
+
+    # Get list of plugin gem names across all scopes to prevent uninstall of gems from GemModel
+    def self.gem_names
+      result = []
+      scopes = ScopeModel.names
+      scopes.each do |scope|
+        plugin_names = self.names(scope: scope)
+        plugin_names.each do |plugin_name|
+          gem_name = plugin_name.split("__")[0]
+          result << gem_name unless result.include?(gem_name)
+        end
+      end
+      return result.sort
+    end
   end
 end

openc3/lib/openc3/utilities/local_mode.rb

@@ -277,7 +277,7 @@ def self.update_local_plugin_files(full_folder_path, plugin_file_path, plugin_ha
       temp_dir = Dir.mktmpdir
       begin
         unless File.exists?(plugin_file_path)
-          plugin_file_path = OpenC3::GemModel.get(temp_dir, plugin_file_path)
+          plugin_file_path = OpenC3::GemModel.get(plugin_file_path)
         end
         File.open(File.join(full_folder_path, File.basename(plugin_file_path)), 'wb') do |file|
           data = File.read(plugin_file_path)

openc3/spec/models/gem_model_spec.rb

@@ -20,20 +20,31 @@
 require 'spec_helper'
 require 'tempfile'
 require 'ostruct'
+require 'openc3/models/plugin_model'
 require 'openc3/models/gem_model'
 require 'openc3/utilities/aws_bucket'
+require 'fileutils'
 
 module OpenC3
   describe GemModel do
     before(:each) do
+      mock_redis()
+      @orig_gem_home = ENV['GEM_HOME']
+      @temp_dir = Dir.mktmpdir
+      ENV['GEM_HOME'] = @temp_dir
       @scope = "DEFAULT"
-      @s3 = instance_double("Aws::S3::Client")
-      @list_result = OpenStruct.new
-      @list_result.contents = [OpenStruct.new({ key: 'openc3-test1.gem' }), OpenStruct.new({ key: 'openc3-test2.gem' })]
-      allow(@s3).to receive(:list_objects_v2).and_return(@list_result)
-      allow(@s3).to receive(:head_bucket).with(any_args)
-      allow(@s3).to receive(:create_bucket)
-      allow(Aws::S3::Client).to receive(:new).and_return(@s3)
+      @gem_list = ['openc3-test1.gem', 'openc3-test2.gem']
+      FileUtils.mkdir_p("#{ENV['GEM_HOME']}/cache")
+      @gem_list.each do |gem|
+        FileUtils.mkdir_p("#{ENV['GEM_HOME']}/gems/#{File.basename(gem, '.gem')}")
+        FileUtils.touch("#{ENV['GEM_HOME']}/cache/#{gem}")
+      end
+    end
+
+    after(:each) do
+      FileUtils.remove_entry(@temp_dir) if @temp_dir and File.exist?(@temp_dir)
+      @temp_dir = nil
+      ENV['GEM_HOME'] = @orig_gem_home
     end
 
     describe "self.names" do
@@ -43,11 +54,9 @@ module OpenC3
     end
 
     describe "self.get" do
-      it "copies the gem to the local filesystem" do
-        response_path = File.join(Dir.pwd, 'openc3-test1.gem')
-        expect(@s3).to receive(:get_object).with(bucket: 'gems', key: 'openc3-test1.gem', response_target: response_path)
-        path = GemModel.get(Dir.pwd, 'openc3-test1.gem')
-        expect(path).to eql response_path
+      it "get the gem on the local filesystem" do
+        path = GemModel.get('openc3-test1.gem')
+        expect(path).to eql "#{ENV['GEM_HOME']}/cache/openc3-test1.gem"
       end
     end
 
@@ -61,15 +70,16 @@ module OpenC3
         expect(pm).to receive_message_chain(:instance, :spawn)
         tf = Tempfile.new("openc3-test3.gem")
         tf.close
-        expect(@s3).to receive(:put_object).with(bucket: 'gems', key: File.basename(tf.path), body: anything, cache_control: nil, content_type: nil, metadata: nil)
         GemModel.put(tf.path, scope: 'DEFAULT')
         tf.unlink
       end
     end
 
     describe "self.destroy" do
       it "removes the gem from the gem server" do
-        expect(@s3).to receive(:delete_object).with(bucket: 'gems', key: 'openc3-test1.gem')
+        uninstaller = instance_double("Gem::Uninstaller").as_null_object
+        expect(Gem::Uninstaller).to receive(:new).and_return(uninstaller)
+        expect(uninstaller).to receive(:uninstall)
         GemModel.destroy("openc3-test1.gem")
       end
     end

openc3/spec/models/plugin_model_spec.rb

@@ -19,6 +19,7 @@
 
 require 'spec_helper'
 require 'openc3/models/plugin_model'
+require 'openc3/utilities/aws_bucket'
 
 module OpenC3
   describe PluginModel do
@@ -104,6 +105,7 @@ module OpenC3
 
         variables = { "folder" => "THE_FOLDER", "name" => "THE_NAME" }
         # Just stub the instance deploy method
+        expect(GemModel).to receive(:install).and_return(nil)
         expect_any_instance_of(ToolModel).to receive(:deploy).with(anything, variables, validate_only: false).and_return(nil)
         expect_any_instance_of(TargetModel).to receive(:deploy).with(anything, variables, validate_only: false).and_return(nil)
         plugin_model = PluginModel.install_phase2({"name" => "name", "variables" => variables, "plugin_txt_lines" => ["TOOL THE_FOLDER THE_NAME", "  URL myurl", "TARGET THE_FOLDER THE_NAME"]}, scope: "DEFAULT")

openc3/spec/spec_helper.rb

@@ -54,7 +54,6 @@
 # Disable Redis and Fluentd in the Logger
 ENV['OPENC3_NO_STORE'] = 'true'
 ENV['OPENC3_LOGS_BUCKET'] = 'logs'
-ENV['OPENC3_GEMS_BUCKET'] = 'gems'
 ENV['OPENC3_TOOLS_BUCKET'] = 'tools'
 ENV['OPENC3_CONFIG_BUCKET'] = 'config'
 # Set some usernames / passwords