diff --git a/lib/embedded_ansible.rb b/lib/embedded_ansible.rb
index 942b874607f..62fad55f0a5 100644
--- a/lib/embedded_ansible.rb
+++ b/lib/embedded_ansible.rb
@@ -2,6 +2,7 @@
 require "awesome_spawn"
 require "linux_admin"
 require "ansible_tower_client"
+require "fileutils"
 
 class EmbeddedAnsible
   ANSIBLE_ROLE           = "embedded_ansible".freeze
@@ -33,7 +34,10 @@ def self.running?
 
   def self.configured?
     return true if MiqEnvironment::Command.is_container?
+
     return false unless File.exist?(SECRET_KEY_FILE)
+    return false unless setup_completed?
+
     key = miq_database.ansible_secret_key
     key.present? && key == File.read(SECRET_KEY_FILE)
   end
@@ -149,9 +153,11 @@ def self.run_setup_script(exclude_tags)
       }
       AwesomeSpawn.run!(SETUP_SCRIPT, :params => params)
     end
+    write_setup_complete_file
   rescue AwesomeSpawn::CommandResultError => e
     _log.error("EmbeddedAnsible setup script failed with: #{e.message}")
     miq_database.ansible_secret_key = nil
+    FileUtils.rm_f(setup_complete_file)
     raise
   end
   private_class_method :run_setup_script
@@ -254,4 +260,19 @@ def self.tower_rpm_version
     LinuxAdmin::Rpm.info("ansible-tower-server")["version"]
   end
   private_class_method :tower_rpm_version
+
+  def self.write_setup_complete_file
+    FileUtils.touch(setup_complete_file)
+  end
+  private_class_method :write_setup_complete_file
+
+  def self.setup_completed?
+    File.exist?(setup_complete_file)
+  end
+  private_class_method :setup_completed?
+
+  def self.setup_complete_file
+    Rails.root.join("tmp", "embedded_ansible_setup_complete")
+  end
+  private_class_method :setup_complete_file
 end
diff --git a/spec/lib/embedded_ansible_spec.rb b/spec/lib/embedded_ansible_spec.rb
index eedef3ae9c6..7e1b73cd04e 100644
--- a/spec/lib/embedded_ansible_spec.rb
+++ b/spec/lib/embedded_ansible_spec.rb
@@ -242,10 +242,12 @@
     end
 
     context "with a key file" do
-      let(:key_file) { Tempfile.new("SECRET_KEY") }
+      let(:key_file)      { Tempfile.new("SECRET_KEY") }
+      let(:complete_file) { Tempfile.new("embedded_ansible_setup_complete") }
 
       before do
         stub_const("EmbeddedAnsible::SECRET_KEY_FILE", key_file.path)
+        allow(described_class).to receive(:setup_complete_file).and_return(complete_file.path)
       end
 
       after do
@@ -262,6 +264,17 @@
           expect(described_class.configured?).to be true
         end
 
+        it "returns false when the key is configured but the complete file is missing" do
+          key = "verysecret"
+          key_file.write(key)
+          key_file.close
+          miq_database.ansible_secret_key = key
+
+          complete_file.unlink
+
+          expect(described_class.configured?).to be false
+        end
+
         it "returns false when there is no key in the database" do
           key_file.write("asdf")
           key_file.close