diff --git a/testdrive/cardano2elastic/.gitignore b/testdrive/cardano2elastic/.gitignore new file mode 100644 index 00000000..4f509e52 --- /dev/null +++ b/testdrive/cardano2elastic/.gitignore @@ -0,0 +1 @@ +*.env \ No newline at end of file diff --git a/testdrive/cardano2elastic/README.md b/testdrive/cardano2elastic/README.md new file mode 100644 index 00000000..0ecd2c64 --- /dev/null +++ b/testdrive/cardano2elastic/README.md @@ -0,0 +1,43 @@ +# Cardano => Elasticsearch Testdrive + +## Introduction + +This is a reference implementation to show how _Oura_ can be leveradged to build read from a Cardano relay node into an Elasticsearch cluster. + +## Prerequisites + +- K8s Cluster +- kubectl +- Skaffold + +## Deployment + +Install Elasticsearch official Kubernetes operator: + +``` +kubectl create -f https://download.elastic.co/downloads/eck/1.9.1/crds.yaml +kubectl apply -f https://download.elastic.co/downloads/eck/1.9.1/operator.yaml +``` + +Create a k8s namespace for the testdrive: + +``` +kubectl create namespace cardano2elastic +``` + +Deploy the Elasticsearch + Kibana resources: + +``` +skaffold run --module elastic --namespace cardano2elastic +``` + +Setup an index template to store _Oura_ events: + +``` +export ELASTIC_AUTH=user:pass +cd scripts && ./setup-index.sh +``` + +``` +skaffold run --namespace cardano2elastic +``` \ No newline at end of file diff --git a/testdrive/cardano2elastic/k8s/elastic.yaml b/testdrive/cardano2elastic/k8s/elastic.yaml new file mode 100644 index 00000000..90c22e94 --- /dev/null +++ b/testdrive/cardano2elastic/k8s/elastic.yaml @@ -0,0 +1,41 @@ +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: sink +spec: + version: 7.16.1 + nodeSets: + - name: default + count: 1 + volumeClaimTemplates: + - metadata: + name: elasticsearch-data # Do not change this name unless you set up a volume mount for the data path. + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + podTemplate: + spec: + containers: + - name: elasticsearch + env: + - name: ES_JAVA_OPTS + value: -Xms2g -Xmx2g + resources: + requests: + memory: 4Gi + cpu: 1 + limits: + memory: 4Gi +--- +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: sink +spec: + version: 7.16.1 + count: 1 + elasticsearchRef: + name: sink \ No newline at end of file diff --git a/testdrive/cardano2elastic/k8s/oura.yaml b/testdrive/cardano2elastic/k8s/oura.yaml new file mode 100644 index 00000000..d0d1dd19 --- /dev/null +++ b/testdrive/cardano2elastic/k8s/oura.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: oura +data: + daemon.toml: |- + [source] + type = "N2N" + address = ["Tcp", "relays-new.cardano-mainnet.iohk.io:3001"] + magic = "mainnet" + + # it will start from way back, so we get a lot of data + since = [4492799, "f8084c61b6a238acec985b59310b6ecec49c0ab8352249afd7268da5cff2a457"] + + [[filters]] + type = "Fingerprint" + + [sink] + type = "Elastic" + url = "https://sink-es-http:9200" + index = "oura.sink.v0.mainnet" + idempotency = true + + [sink.credentials] + type = "Basic" + username = "elastic" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oura + labels: + app: oura +spec: + replicas: 1 + selector: + matchLabels: + app: oura + template: + metadata: + labels: + app: oura + spec: + containers: + - name: main + image: ghcr.io/txpipe/oura:v0.3.10 + env: + - name: "RUST_LOG" + value: "info" + - name: "OURA_SINK_CREDENTIALS_PASSWORD" + valueFrom: + secretKeyRef: + key: elastic + name: sink-es-elastic-user + resources: + requests: + memory: 100Mi + cpu: 50m + limits: + memory: 500Mi + cpu: 200m + args: + - "daemon" + volumeMounts: + - mountPath: /etc/oura + name: oura-config + volumes: + - name: oura-config + configMap: + name: oura diff --git a/testdrive/cardano2elastic/scripts/index-template.json b/testdrive/cardano2elastic/scripts/index-template.json new file mode 100644 index 00000000..4e09b2b2 --- /dev/null +++ b/testdrive/cardano2elastic/scripts/index-template.json @@ -0,0 +1,261 @@ +{ + "template": { + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "block": { + "properties": { + "tx_count": { + "type": "long" + }, + "body_size": { + "type": "long" + }, + "issuer_vkey": { + "type": "keyword" + } + } + }, + "collateral": { + "properties": { + "index": { + "type": "long" + }, + "tx_id": { + "type": "keyword" + } + } + }, + "context": { + "properties": { + "block_number": { + "type": "long" + }, + "block_hash": { + "type": "keyword" + }, + "input_idx": { + "type": "long" + }, + "output_idx": { + "type": "long" + }, + "certificate_idx": { + "type": "long" + }, + "slot": { + "type": "long" + }, + "timestamp": { + "type": "long" + }, + "tx_hash": { + "type": "keyword" + }, + "tx_idx": { + "type": "long" + } + } + }, + "metadata": { + "properties": { + "label": { + "type": "keyword" + }, + "map_json": { + "enabled": false + }, + "array_json": { + "enabled": false + }, + "bytes_hex": { + "enabled": false + }, + "text_scalar": { + "type": "text" + }, + "int_scalar": { + "type": "long" + } + } + }, + "cip25_asset": { + "properties": { + "policy": { + "type": "keyword" + }, + "asset": { + "type": "keyword" + }, + "name": { + "type": "text" + }, + "mediatType": { + "type": "keyword" + }, + "image": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "raw_json": { + "enabled": false + } + } + }, + "mint": { + "properties": { + "asset": { + "type": "keyword" + }, + "policy": { + "type": "keyword" + }, + "quantity": { + "type": "long" + } + } + }, + "output_asset": { + "properties": { + "amount": { + "type": "long" + }, + "asset": { + "type": "keyword" + }, + "policy": { + "type": "keyword" + } + } + }, + "roll_back": { + "properties": { + "block_hash": { + "type": "keyword" + }, + "block_slot": { + "type": "long" + } + } + }, + "stake_delegation": { + "properties": { + "credential": { + "properties": { + "AddrKeyhash": { + "type": "keyword" + } + } + }, + "pool_hash": { + "type": "keyword" + } + } + }, + "stake_deregistration": { + "properties": { + "credential": { + "properties": { + "AddrKeyhash": { + "type": "keyword" + } + } + } + } + }, + "stake_registration": { + "properties": { + "credential": { + "properties": { + "AddrKeyhash": { + "type": "keyword" + } + } + } + } + }, + "transaction": { + "properties": { + "fee": { + "type": "long" + }, + "ttl": { + "type": "long" + }, + "validity_interval_start": { + "type": "long" + }, + "input_count": { + "type": "long" + }, + "output_count": { + "type": "long" + }, + "total_output": { + "type": "long" + }, + "metadata": { + "type": "object", + "properties": { + "label": { + "type": "keyword" + }, + "map_json": { + "enabled": false + }, + "array_json": { + "enabled": false + }, + "bytes_hex": { + "enabled": false + }, + "text_scalar": { + "type": "text" + }, + "int_scalar": { + "type": "long" + } + } + } + } + }, + "tx_input": { + "properties": { + "index": { + "type": "long" + }, + "tx_id": { + "type": "keyword" + } + } + }, + "tx_output": { + "properties": { + "address": { + "type": "keyword" + }, + "amount": { + "type": "long" + } + } + }, + "fingerprint": { + "type": "keyword", + "ignore_above": 256 + }, + "variant": { + "type": "keyword", + "ignore_above": 256 + } + } + } + }, + "data_stream": {}, + "index_patterns": [ + "oura.sink.v0.*" + ] +} \ No newline at end of file diff --git a/testdrive/cardano2elastic/scripts/setup-index.sh b/testdrive/cardano2elastic/scripts/setup-index.sh new file mode 100755 index 00000000..f4a997b3 --- /dev/null +++ b/testdrive/cardano2elastic/scripts/setup-index.sh @@ -0,0 +1,4 @@ +curl -k -X PUT https://localhost:9200/_index_template/oura.sink.v0 \ + -u ${ELASTIC_AUTH} \ + -H 'Content-Type: application/json' \ + -d @index-template.json diff --git a/testdrive/cardano2elastic/skaffold.yaml b/testdrive/cardano2elastic/skaffold.yaml new file mode 100644 index 00000000..4b4c70e5 --- /dev/null +++ b/testdrive/cardano2elastic/skaffold.yaml @@ -0,0 +1,19 @@ +apiVersion: skaffold/v2beta26 +kind: Config +metadata: + name: elastic +deploy: + kubectl: + manifests: + - k8s/elastic.yaml +--- +apiVersion: skaffold/v2beta26 +kind: Config +metadata: + name: oura +requires: + - configs: [elastic] +deploy: + kubectl: + manifests: + - k8s/oura.yaml