diff --git a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
index dc144c13d50a4..1d0262035f01d 100644
--- a/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
+++ b/core/src/main/java/org/elasticsearch/index/shard/IndexShard.java
@@ -872,7 +872,7 @@ public DocsStats docStats() {
         long numDeletedDocs = 0;
         long sizeInBytes = 0;
         try (Engine.Searcher searcher = acquireSearcher("docStats", Engine.SearcherScope.INTERNAL)) {
-            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accesssed only which will cause
+            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
             // the next scheduled refresh to go through and refresh the stats as well
             markSearcherAccessed();
             for (LeafReaderContext reader : searcher.reader().leaves()) {
@@ -971,7 +971,7 @@ public TranslogStats translogStats() {
     public CompletionStats completionStats(String... fields) {
         CompletionStats completionStats = new CompletionStats();
         try (Engine.Searcher currentSearcher = acquireSearcher("completion_stats")) {
-            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accesssed only which will cause
+            // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
             // the next scheduled refresh to go through and refresh the stats as well
             markSearcherAccessed();
             completionStats.add(CompletionFieldStats.completionStats(currentSearcher.reader(), fields));
@@ -2456,7 +2456,9 @@ public boolean scheduledRefresh() {
         boolean listenerNeedsRefresh = refreshListeners.refreshNeeded();
         if (isReadAllowed() && (listenerNeedsRefresh || getEngine().refreshNeeded())) {
             if (listenerNeedsRefresh == false // if we have a listener that is waiting for a refresh we need to force it
-                && isSearchIdle() && indexSettings.isExplicitRefresh() == false) {
+                && isSearchIdle()
+                && indexSettings.isExplicitRefresh() == false
+                && active.get()) { // it must be active otherwise we might not free up segment memory once the shard became inactive
                 // lets skip this refresh since we are search idle and
                 // don't necessarily need to refresh. the next searcher access will register a refreshListener and that will
                 // cause the next schedule to refresh.
diff --git a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
index fe6d4ccdea08a..e17e4afd3c942 100644
--- a/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
+++ b/core/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java
@@ -2684,6 +2684,15 @@ public void testScheduledRefresh() throws IOException, InterruptedException {
 
         });
         latch1.await();
+
+        indexDoc(primary, "test", "2", "{\"foo\" : \"bar\"}");
+        assertFalse(primary.scheduledRefresh());
+        assertTrue(primary.isSearchIdle());
+        primary.checkIdle(0);
+        assertTrue(primary.scheduledRefresh()); // make sure we refresh once the shard is inactive
+        try (Engine.Searcher searcher = primary.acquireSearcher("test")) {
+            assertEquals(3, searcher.reader().numDocs());
+        }
         closeShards(primary);
     }