Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Elasticsearch exception [type=version_conflict_engine_exception #3315

Closed
1 task
wuxingye opened this issue Aug 25, 2019 · 1 comment
Closed
1 task

Elasticsearch exception [type=version_conflict_engine_exception #3315

wuxingye opened this issue Aug 25, 2019 · 1 comment
Labels
question End user question and discussion.
Milestone
6.4.0

Comments

@wuxingye
Copy link
Contributor

Please answer these questions before submitting your issue.

  • Why do you submit this issue?

  • Bug

Bug

  • Which version of SkyWalking, OS and JRE?

SkyWalking: 6.3

Elasticsearch:6.8.2

OS and JRE: official docker images

mode: Zookeeper cluster

detail (docker-compose.yml):

version: '3.3'

services:
  es6_01:
    restart: always
    container_name: es6_01
    image: elasticsearch:6.8.2
    hostname: es6_01
    privileged: true
    environment:
      - TZ="Asia/Shanghai"
      - node.name=es6_01
      - network.host=es6_01
      - discovery.zen.minimum_master_nodes=2
      - discovery.zen.ping.unicast.hosts=es6_01,es6_02,es6_03
      - cluster.name=es6
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - es6_data1:/usr/share/elasticsearch/data
    ports:
      - 9211:9200
      - 9311:9300

  es6_02:
    restart: always
    container_name: es6_02
    image: elasticsearch:6.8.2
    hostname: es6_02
    privileged: true
    environment:
      - TZ="Asia/Shanghai"
      - node.name=es6_02
      - network.host=es6_02
      - discovery.zen.minimum_master_nodes=2
      - discovery.zen.ping.unicast.hosts=es6_01,es6_02,es6_03
      - cluster.name=es6
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - es6_data2:/usr/share/elasticsearch/data
    ports:
      - 9212:9200
      - 9312:9300

  es6_03:
    restart: always
    container_name: es6_03
    image: elasticsearch:6.8.2
    hostname: es6_03
    privileged: true
    environment:
      - TZ="Asia/Shanghai"
      - node.name=es6_03
      - network.host=es6_03
      - discovery.zen.minimum_master_nodes=2
      - discovery.zen.ping.unicast.hosts=es6_01,es6_02,es6_03
      - cluster.name=es6
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - es6_data3:/usr/share/elasticsearch/data
    ports:
      - 9213:9200
      - 9313:9300

  oap1:
    image: apache/skywalking-oap-server:6.3.0
    container_name: oap1
    hostname: oap1
    privileged: true
    restart: always
    ports:
      - 11801:11800
      - 12801:12800
    environment:
      TZ: Asia/Shanghai
      SW_CLUSTER: zookeeper
      SW_CLUSTER_ZK_HOST_PORT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      SW_STORAGE: elasticsearch
      SW_STORAGE_ES_CLUSTER_NODES: es6_01:9200,es6_02:9200,es6_03:9200
    depends_on:
      - es6_01
      - es6_02
      - es6_03
    links:
      - es6_01
      - es6_02
      - es6_03
  oap2:
    image: apache/skywalking-oap-server:6.3.0
    container_name: oap2
    hostname: oap2
    privileged: true
    restart: always
    ports:
      - 11802:11800
      - 12802:12800
    environment:
      TZ: Asia/Shanghai
      SW_CLUSTER: zookeeper
      SW_CLUSTER_ZK_HOST_PORT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      SW_STORAGE: elasticsearch
      SW_STORAGE_ES_CLUSTER_NODES: es6_01:9200,es6_02:9200,es6_03:9200
    depends_on:
      - es6_01
      - es6_02
      - es6_03
    links:
      - es6_01
      - es6_02
      - es6_03
  oap3:
    image: apache/skywalking-oap-server:6.3.0
    container_name: oap3
    hostname: oap3
    privileged: true
    restart: always
    ports:
      - 11803:11800
      - 12803:12800
    environment:
      TZ: Asia/Shanghai
      SW_CLUSTER: zookeeper
      SW_CLUSTER_ZK_HOST_PORT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      SW_STORAGE: elasticsearch
      SW_STORAGE_ES_CLUSTER_NODES: es6_01:9200,es6_02:9200,es6_03:9200
    depends_on:
      - es6_01
      - es6_02
      - es6_03
    links:
      - es6_01
      - es6_02
      - es6_03

  ui:
    image: apache/skywalking-ui:6.3.0
    container_name: ui
    depends_on:
      - oap1
      - oap2
      - oap3
    links:
      - oap1
      - oap2
      - oap3
    restart: always
    ports:
      - 2800:8080
    environment:
      SW_OAP_ADDRESS: oap1:12800,oap2:12800,oap3:12800

networks:
  default:
    external:
      name: kakarot

volumes:
  es6_data1:
    driver_opts:
      type: none
      device: /data/elasticsearch6/data1
      o: bind
  es6_data2:
    driver_opts:
      type: none
      device: /data/elasticsearch6/data2
      o: bind
  es6_data3:
    driver_opts:
      type: none
      device: /data/elasticsearch6/data3
      o: bind
  • Which company or project?

Thomas Cook

  • What happen?
    If possible, provide a way for reproducing the error. e.g. demo application, component version.

error below:

2019-08-25 08:31:32,551 - org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker -48014893 [DataCarrier.REGISTER_L2.BulkConsumePool.0.Thread] ERROR [] - Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]]
org.elasticsearch.ElasticsearchStatusException: Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]]
	at org.elasticsearch.rest.BytesRestResponse.errorFromXContent(BytesRestResponse.java:177) ~[elasticsearch-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.parseEntity(RestHighLevelClient.java:653) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.parseResponseException(RestHighLevelClient.java:628) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:535) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:508) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.elasticsearch.client.RestHighLevelClient.update(RestHighLevelClient.java:366) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
	at org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient.forceUpdate(ElasticSearchClient.java:254) ~[library-client-6.3.0.jar:6.3.0]
	at org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.RegisterEsDAO.forceUpdate(RegisterEsDAO.java:56) ~[storage-elasticsearch-plugin-6.3.0.jar:6.3.0]
	at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.lambda$onWork$0(RegisterPersistentWorker.java:89) ~[server-core-6.3.0.jar:6.3.0]
	at java.util.HashMap$Values.forEach(HashMap.java:981) [?:1.8.0_181]
	at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.onWork(RegisterPersistentWorker.java:84) [server-core-6.3.0.jar:6.3.0]
	at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.access$100(RegisterPersistentWorker.java:35) [server-core-6.3.0.jar:6.3.0]
	at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker$PersistentConsumer.consume(RegisterPersistentWorker.java:141) [server-core-6.3.0.jar:6.3.0]
	at org.apache.skywalking.apm.commons.datacarrier.consumer.MultipleChannelsConsumer.consume(MultipleChannelsConsumer.java:82) [apm-datacarrier-6.3.0.jar:6.3.0]
	at org.apache.skywalking.apm.commons.datacarrier.consumer.MultipleChannelsConsumer.run(MultipleChannelsConsumer.java:53) [apm-datacarrier-6.3.0.jar:6.3.0]
	Suppressed: org.elasticsearch.client.ResponseException: method [POST], host [http://es6_01:9200], URI [/service_inventory/type/SkyWalking_0_0/_update?refresh=true&timeout=1m], status line [HTTP/1.1 409 Conflict]
{"error":{"root_cause":[{"type":"version_conflict_engine_exception","reason":"[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]","index_uuid":"PQvASg4pRtSElaKDAc5KwA","shard":"0","index":"service_inventory"}],"type":"version_conflict_engine_exception","reason":"[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]","index_uuid":"PQvASg4pRtSElaKDAc5KwA","shard":"0","index":"service_inventory"},"status":409}
		at org.elasticsearch.client.RestClient$SyncResponseListener.get(RestClient.java:705) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestClient.performRequest(RestClient.java:235) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestClient.performRequest(RestClient.java:198) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestHighLevelClient.performRequest(RestHighLevelClient.java:522) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestHighLevelClient.performRequestAndParseEntity(RestHighLevelClient.java:508) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestHighLevelClient.update(RestHighLevelClient.java:366) ~[elasticsearch-rest-high-level-client-6.3.2.jar:6.3.2]
		at org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient.forceUpdate(ElasticSearchClient.java:254) ~[library-client-6.3.0.jar:6.3.0]
		at org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.RegisterEsDAO.forceUpdate(RegisterEsDAO.java:56) ~[storage-elasticsearch-plugin-6.3.0.jar:6.3.0]
		at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.lambda$onWork$0(RegisterPersistentWorker.java:89) ~[server-core-6.3.0.jar:6.3.0]
		at java.util.HashMap$Values.forEach(HashMap.java:981) [?:1.8.0_181]
		at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.onWork(RegisterPersistentWorker.java:84) [server-core-6.3.0.jar:6.3.0]
		at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker.access$100(RegisterPersistentWorker.java:35) [server-core-6.3.0.jar:6.3.0]
		at org.apache.skywalking.oap.server.core.register.worker.RegisterPersistentWorker$PersistentConsumer.consume(RegisterPersistentWorker.java:141) [server-core-6.3.0.jar:6.3.0]
		at org.apache.skywalking.apm.commons.datacarrier.consumer.MultipleChannelsConsumer.consume(MultipleChannelsConsumer.java:82) [apm-datacarrier-6.3.0.jar:6.3.0]
		at org.apache.skywalking.apm.commons.datacarrier.consumer.MultipleChannelsConsumer.run(MultipleChannelsConsumer.java:53) [apm-datacarrier-6.3.0.jar:6.3.0]
	Caused by: org.elasticsearch.client.ResponseException: method [POST], host [http://es6_01:9200], URI [/service_inventory/type/SkyWalking_0_0/_update?refresh=true&timeout=1m], status line [HTTP/1.1 409 Conflict]
{"error":{"root_cause":[{"type":"version_conflict_engine_exception","reason":"[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]","index_uuid":"PQvASg4pRtSElaKDAc5KwA","shard":"0","index":"service_inventory"}],"type":"version_conflict_engine_exception","reason":"[type][SkyWalking_0_0]: version conflict, required seqNo [36541], primary term [1]. current document has seqNo [36544] and primary term [1]","index_uuid":"PQvASg4pRtSElaKDAc5KwA","shard":"0","index":"service_inventory"},"status":409}
		at org.elasticsearch.client.RestClient$1.completed(RestClient.java:377) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.elasticsearch.client.RestClient$1.completed(RestClient.java:366) ~[elasticsearch-rest-client-6.3.2.jar:6.3.2]
		at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:119) ~[httpcore-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.responseCompleted(DefaultClientExchangeHandlerImpl.java:177) ~[httpasyncclient-4.1.2.jar:4.1.2]
		at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.processResponse(HttpAsyncRequestExecutor.java:436) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.inputReady(HttpAsyncRequestExecutor.java:326) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:265) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:81) ~[httpasyncclient-4.1.2.jar:4.1.2]
		at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:39) ~[httpasyncclient-4.1.2.jar:4.1.2]
		at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:114) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) ~[httpcore-nio-4.4.5.jar:4.4.5]
		at java.lang.Thread.run(Thread.java:748) ~[?:1.8.0_181]

Requirement or improvement

  • Please describe about your requirements or improvement suggestions.

how to set up a zookeeper based skywalking cluster?
@kezhenxu94
Copy link
Member

Duplicate of #2496 , please refer to that issue for answer

@kezhenxu94 kezhenxu94 added the question End user question and discussion. label Aug 25, 2019
@kezhenxu94 kezhenxu94 added this to the 6.4.0 milestone Aug 25, 2019
@kezhenxu94 kezhenxu94 mentioned this issue Aug 25, 2019
Closed
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
question End user question and discussion.
Projects
None yet
Milestone
6.4.0
Development

No branches or pull requests
2 participants
@wuxingye @kezhenxu94