Why is my Elasticsearch index using so much disk space?

[bartvastgoed]

Elasticsearch version (bin/elasticsearch --version):
5.6.11
Plugins installed: []

JVM version (java -version):
openjdk version "1.8.0_222"
OpenJDK Runtime Environment (build 1.8.0_222-8u222-b10-1ubuntu1~16.04.1-b10)
OpenJDK 64-Bit Server VM (build 25.222-b10, mixed mode)

OS version (uname -a if on a Unix-like system):
Linux myserver 4.4.0-131-generic #157-Ubuntu SMP Thu Jul 12 15:51:36 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

Description of the problem including expected versus actual behavior:
Currently im running into a disk space issue on my server and seem to have found the culprit: Supposibly according to some people the size of this index is ~ 40 gb. Which sounds ridiculous because the average list of coords that I save is about 5 elements.

This index has 11k documents.

What I have tried:

• Deleted the index and filled it up again, this resulted in a MUCH MUCH smaller index on elasticsearch 40GB -> 14MB?? Which is why im very skeptical as to if this will work at all.
• Tried to research trough google without any concrete solutions.

I really hope someone might have some insight why it could possibly be this big.

my settings:

{
  "my_polygons_1": {
    "settings": {
      "index": {
        "number_of_shards": "5",
        "provided_name": "my_polygons_1",
        "creation_date": "1514988794703",
        "analysis": {
          "filter": {
            "addresstranslation": {
              "type": "synonym",
              "synonyms": [
                "1e,eerste",
                "2e,tweede",
                "3e,derde",
                "1ste,eerste",
                "2de,tweede",
                "3de,derde"
              ]
            },
            "translation": {
              "token_chars": [
                "letter",
                "digit",
                "whitespace"
              ],
              "min_gram": "3",
              "type": "nGram",
              "max_gram": "50"
            }
          },
          "analyzer": {
            "translation_index_analyzer": {
              "filter": "standard,lowercase,asciifolding,translation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "address_search_analyzer": {
              "filter": "standard,lowercase,asciifolding,addresstranslation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "address_index_analyzer": {
              "filter": "standard,lowercase,asciifolding,addresstranslation,translation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "translation_search_analyzer": {
              "filter": "standard,lowercase,asciifolding",
              "type": "custom",
              "tokenizer": "standard"
            }
          }
        },
        "number_of_replicas": "1",
        "uuid": "lQPnGOZ6TdKx9xZwNPfs2g",
        "version": {
          "created": "5060399"
        }
      }
    }
  }
}

my _stats:

{
  "_shards": {
    "total": 10,
    "successful": 10,
    "failed": 0
  },
  "_all": {
    "primaries": {
      "docs": {
        "count": 11331,
        "deleted": 646
      },
      "store": {
        "size_in_bytes": 22095974914,
        "throttle_time_in_millis": 0
      },
      "indexing": {
        "index_total": 1184,
        "index_time_in_millis": 1166437,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 368507,
        "time_in_millis": 159070,
        "exists_total": 354156,
        "exists_time_in_millis": 157586,
        "missing_total": 14351,
        "missing_time_in_millis": 1484,
        "current": 0
      },
      "search": {
        "open_contexts": 0,
        "query_total": 185,
        "query_time_in_millis": 46,
        "query_current": 0,
        "fetch_total": 30,
        "fetch_time_in_millis": 89,
        "fetch_current": 0,
        "scroll_total": 2,
        "scroll_time_in_millis": 2444,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size_in_bytes": 0,
        "total": 132,
        "total_time_in_millis": 714437,
        "total_docs": 6268,
        "total_size_in_bytes": 9036940847,
        "total_stopped_time_in_millis": 0,
        "total_throttled_time_in_millis": 88138,
        "total_auto_throttle_in_bytes": 73181749
      },
      "refresh": {
        "total": 3971,
        "total_time_in_millis": 1052798,
        "listeners": 0
      },
      "flush": {
        "total": 1063,
        "total_time_in_millis": 10994
      },
      "warmer": {
        "current": 0,
        "total": 3474,
        "total_time_in_millis": 55
      },
      "query_cache": {
        "memory_size_in_bytes": 0,
        "total_count": 0,
        "hit_count": 0,
        "miss_count": 0,
        "cache_size": 0,
        "cache_count": 0,
        "evictions": 0
      },
      "fielddata": {
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": {
        "size_in_bytes": 0
      },
      "segments": {
        "count": 123,
        "memory_in_bytes": 597667360,
        "terms_memory_in_bytes": 597553396,
        "stored_fields_memory_in_bytes": 38640,
        "term_vectors_memory_in_bytes": 0,
        "norms_memory_in_bytes": 7872,
        "points_memory_in_bytes": 368,
        "doc_values_memory_in_bytes": 67084,
        "index_writer_memory_in_bytes": 0,
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": {}
      },
      "translog": {
        "operations": 0,
        "size_in_bytes": 215
      },
      "request_cache": {
        "memory_size_in_bytes": 0,
        "evictions": 8,
        "hit_count": 0,
        "miss_count": 100
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time_in_millis": 0
      }
    },
    "total": {
      "docs": {
        "count": 22662,
        "deleted": 2356
      },
      "store": {
        "size_in_bytes": 46187867347,
        "throttle_time_in_millis": 0
      },
      "indexing": {
        "index_total": 2263,
        "index_time_in_millis": 2216831,
        "index_current": 0,
        "index_failed": 0,
        "delete_total": 0,
        "delete_time_in_millis": 0,
        "delete_current": 0,
        "noop_update_total": 0,
        "is_throttled": false,
        "throttle_time_in_millis": 0
      },
      "get": {
        "total": 657621,
        "time_in_millis": 250947,
        "exists_total": 628867,
        "exists_time_in_millis": 247657,
        "missing_total": 28754,
        "missing_time_in_millis": 3290,
        "current": 0
      },
      "search": {
        "open_contexts": 0,
        "query_total": 375,
        "query_time_in_millis": 128,
        "query_current": 0,
        "fetch_total": 73,
        "fetch_time_in_millis": 219,
        "fetch_current": 0,
        "scroll_total": 5,
        "scroll_time_in_millis": 6110,
        "scroll_current": 0,
        "suggest_total": 0,
        "suggest_time_in_millis": 0,
        "suggest_current": 0
      },
      "merges": {
        "current": 0,
        "current_docs": 0,
        "current_size_in_bytes": 0,
        "total": 252,
        "total_time_in_millis": 1134358,
        "total_docs": 10222,
        "total_size_in_bytes": 13797985517,
        "total_stopped_time_in_millis": 0,
        "total_throttled_time_in_millis": 160514,
        "total_auto_throttle_in_bytes": 155271619
      },
      "refresh": {
        "total": 7539,
        "total_time_in_millis": 1978415,
        "listeners": 0
      },
      "flush": {
        "total": 2030,
        "total_time_in_millis": 21846
      },
      "warmer": {
        "current": 0,
        "total": 6641,
        "total_time_in_millis": 105
      },
      "query_cache": {
        "memory_size_in_bytes": 0,
        "total_count": 0,
        "hit_count": 0,
        "miss_count": 0,
        "cache_size": 0,
        "cache_count": 0,
        "evictions": 0
      },
      "fielddata": {
        "memory_size_in_bytes": 0,
        "evictions": 0
      },
      "completion": {
        "size_in_bytes": 0
      },
      "segments": {
        "count": 261,
        "memory_in_bytes": 1246021155,
        "terms_memory_in_bytes": 1245778862,
        "stored_fields_memory_in_bytes": 81992,
        "term_vectors_memory_in_bytes": 0,
        "norms_memory_in_bytes": 16704,
        "points_memory_in_bytes": 769,
        "doc_values_memory_in_bytes": 142828,
        "index_writer_memory_in_bytes": 0,
        "version_map_memory_in_bytes": 0,
        "fixed_bit_set_memory_in_bytes": 0,
        "max_unsafe_auto_id_timestamp": -1,
        "file_sizes": {}
      },
      "translog": {
        "operations": 0,
        "size_in_bytes": 430
      },
      "request_cache": {
        "memory_size_in_bytes": 0,
        "evictions": 16,
        "hit_count": 0,
        "miss_count": 200
      },
      "recovery": {
        "current_as_source": 0,
        "current_as_target": 0,
        "throttle_time_in_millis": 0
      }
    }
  },
  "indices": {
    "my_polygons_1": {
      "primaries": {
        "docs": {
          "count": 11331,
          "deleted": 646
        },
        "store": {
          "size_in_bytes": 22095974914,
          "throttle_time_in_millis": 0
        },
        "indexing": {
          "index_total": 1184,
          "index_time_in_millis": 1166437,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 368507,
          "time_in_millis": 159070,
          "exists_total": 354156,
          "exists_time_in_millis": 157586,
          "missing_total": 14351,
          "missing_time_in_millis": 1484,
          "current": 0
        },
        "search": {
          "open_contexts": 0,
          "query_total": 185,
          "query_time_in_millis": 46,
          "query_current": 0,
          "fetch_total": 30,
          "fetch_time_in_millis": 89,
          "fetch_current": 0,
          "scroll_total": 2,
          "scroll_time_in_millis": 2444,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size_in_bytes": 0,
          "total": 132,
          "total_time_in_millis": 714437,
          "total_docs": 6268,
          "total_size_in_bytes": 9036940847,
          "total_stopped_time_in_millis": 0,
          "total_throttled_time_in_millis": 88138,
          "total_auto_throttle_in_bytes": 73181749
        },
        "refresh": {
          "total": 3971,
          "total_time_in_millis": 1052798,
          "listeners": 0
        },
        "flush": {
          "total": 1063,
          "total_time_in_millis": 10994
        },
        "warmer": {
          "current": 0,
          "total": 3474,
          "total_time_in_millis": 55
        },
        "query_cache": {
          "memory_size_in_bytes": 0,
          "total_count": 0,
          "hit_count": 0,
          "miss_count": 0,
          "cache_size": 0,
          "cache_count": 0,
          "evictions": 0
        },
        "fielddata": {
          "memory_size_in_bytes": 0,
          "evictions": 0
        },
        "completion": {
          "size_in_bytes": 0
        },
        "segments": {
          "count": 123,
          "memory_in_bytes": 597667360,
          "terms_memory_in_bytes": 597553396,
          "stored_fields_memory_in_bytes": 38640,
          "term_vectors_memory_in_bytes": 0,
          "norms_memory_in_bytes": 7872,
          "points_memory_in_bytes": 368,
          "doc_values_memory_in_bytes": 67084,
          "index_writer_memory_in_bytes": 0,
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set_memory_in_bytes": 0,
          "max_unsafe_auto_id_timestamp": -1,
          "file_sizes": {}
        },
        "translog": {
          "operations": 0,
          "size_in_bytes": 215
        },
        "request_cache": {
          "memory_size_in_bytes": 0,
          "evictions": 8,
          "hit_count": 0,
          "miss_count": 100
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time_in_millis": 0
        }
      },
      "total": {
        "docs": {
          "count": 22662,
          "deleted": 2356
        },
        "store": {
          "size_in_bytes": 46187867347,
          "throttle_time_in_millis": 0
        },
        "indexing": {
          "index_total": 2263,
          "index_time_in_millis": 2216831,
          "index_current": 0,
          "index_failed": 0,
          "delete_total": 0,
          "delete_time_in_millis": 0,
          "delete_current": 0,
          "noop_update_total": 0,
          "is_throttled": false,
          "throttle_time_in_millis": 0
        },
        "get": {
          "total": 657621,
          "time_in_millis": 250947,
          "exists_total": 628867,
          "exists_time_in_millis": 247657,
          "missing_total": 28754,
          "missing_time_in_millis": 3290,
          "current": 0
        },
        "search": {
          "open_contexts": 0,
          "query_total": 375,
          "query_time_in_millis": 128,
          "query_current": 0,
          "fetch_total": 73,
          "fetch_time_in_millis": 219,
          "fetch_current": 0,
          "scroll_total": 5,
          "scroll_time_in_millis": 6110,
          "scroll_current": 0,
          "suggest_total": 0,
          "suggest_time_in_millis": 0,
          "suggest_current": 0
        },
        "merges": {
          "current": 0,
          "current_docs": 0,
          "current_size_in_bytes": 0,
          "total": 252,
          "total_time_in_millis": 1134358,
          "total_docs": 10222,
          "total_size_in_bytes": 13797985517,
          "total_stopped_time_in_millis": 0,
          "total_throttled_time_in_millis": 160514,
          "total_auto_throttle_in_bytes": 155271619
        },
        "refresh": {
          "total": 7539,
          "total_time_in_millis": 1978415,
          "listeners": 0
        },
        "flush": {
          "total": 2030,
          "total_time_in_millis": 21846
        },
        "warmer": {
          "current": 0,
          "total": 6641,
          "total_time_in_millis": 105
        },
        "query_cache": {
          "memory_size_in_bytes": 0,
          "total_count": 0,
          "hit_count": 0,
          "miss_count": 0,
          "cache_size": 0,
          "cache_count": 0,
          "evictions": 0
        },
        "fielddata": {
          "memory_size_in_bytes": 0,
          "evictions": 0
        },
        "completion": {
          "size_in_bytes": 0
        },
        "segments": {
          "count": 261,
          "memory_in_bytes": 1246021155,
          "terms_memory_in_bytes": 1245778862,
          "stored_fields_memory_in_bytes": 81992,
          "term_vectors_memory_in_bytes": 0,
          "norms_memory_in_bytes": 16704,
          "points_memory_in_bytes": 769,
          "doc_values_memory_in_bytes": 142828,
          "index_writer_memory_in_bytes": 0,
          "version_map_memory_in_bytes": 0,
          "fixed_bit_set_memory_in_bytes": 0,
          "max_unsafe_auto_id_timestamp": -1,
          "file_sizes": {}
        },
        "translog": {
          "operations": 0,
          "size_in_bytes": 430
        },
        "request_cache": {
          "memory_size_in_bytes": 0,
          "evictions": 16,
          "hit_count": 0,
          "miss_count": 200
        },
        "recovery": {
          "current_as_source": 0,
          "current_as_target": 0,
          "throttle_time_in_millis": 0
        }
      }
    }
  }
}

and my mapping:

{
  "my_polygons_1": {
    "aliases": {},
    "mappings": {
      "my_polygons_1": {
        "properties": {
          "client_id": {
            "type": "integer"
          },
          "id": {
            "type": "integer"
          },
          "name": {
            "type": "keyword"
          },
          "polygon": {
            "type": "geo_shape",
            "tree": "quadtree",
            "precision": "1.0m"
          }
        }
      }
    },
    "settings": {
      "index": {
        "number_of_shards": "5",
        "provided_name": "my_polygons_1",
        "creation_date": "1514988794703",
        "analysis": {
          "filter": {
            "addresstranslation": {
              "type": "synonym",
              "synonyms": [
                "1e,eerste",
                "2e,tweede",
                "3e,derde",
                "1ste,eerste",
                "2de,tweede",
                "3de,derde"
              ]
            },
            "translation": {
              "token_chars": [
                "letter",
                "digit",
                "whitespace"
              ],
              "min_gram": "3",
              "type": "nGram",
              "max_gram": "50"
            }
          },
          "analyzer": {
            "translation_index_analyzer": {
              "filter": "standard,lowercase,asciifolding,translation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "address_search_analyzer": {
              "filter": "standard,lowercase,asciifolding,addresstranslation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "address_index_analyzer": {
              "filter": "standard,lowercase,asciifolding,addresstranslation,translation",
              "type": "custom",
              "tokenizer": "standard"
            },
            "translation_search_analyzer": {
              "filter": "standard,lowercase,asciifolding",
              "type": "custom",
              "tokenizer": "standard"
            }
          }
        },
        "number_of_replicas": "1",
        "uuid": "lQPnGOZ6TdKx9xZwNPfs2g",
        "version": {
          "created": "5060399"
        }
      }
    }
  }
}

[ywelsch]

Thanks very much for your interest in Elasticsearch.

This appears to be a user question, and we'd like to direct these kinds of things to the forums. If you can stop by there, we'd appreciate it. This allows us to use GitHub for verified bug reports, feature requests, and pull requests.

There's an active community in the forums that should be able to help get an answer to your question. As such, I hope you don't mind that I close this.