Skip to main content
Version: 6.4

Multi-Tier Cluster

The Multi-Tier Cluster (x-large) deployment is suitable for high ingest rates, while also supporting longer retention periods.

Sizing ParameterValue
Licensed Unitsup to 16
Recommended Max. Ingest Rate48000 flows/sec
Retention at Max. Rate30 days
Shards6
Replicas1

This multi-tier architecture provides hot data nodes for high-performance ingest, as well as warm nodes for increased storage capacity at a lower cost. Additional warm nodes can be added for even greater retention periods.

ApplicationCPU CoresMemoryStorage
Kibana, Elasticsearch (coordinating)832 GB128 GB
ApplicationCPU CoresMemoryStorage
Elasticsearch (master)432 GB128 GB
Elasticsearch (master)432 GB128 GB
Elasticsearch (master)432 GB128 GB
ApplicationCPU CoresMemorySSD Storage
Elasticsearch (hot data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (hot data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (hot data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (hot data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (hot data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (hot data)12-1664 GB2 x 4 TB (6.8 TB)
ApplicationCPU CoresMemoryHDD Storage
Elasticsearch (warm data)8-1664 GB4 x 4 TB (13.6 TB)
Elasticsearch (warm data)8-1664 GB4 x 4 TB (13.6 TB)
Elasticsearch (warm data)8-1664 GB4 x 4 TB (13.6 TB)
Elasticsearch (warm data)8-1664 GB4 x 4 TB (13.6 TB)
Elasticsearch (warm data)8-1664 GB4 x 4 TB (13.6 TB)
Elasticsearch (warm data)8-1664 GB4 x 4 TB (13.6 TB)
ApplicationCPU CoresMemoryStorage
Flow Collector, Elasticsearch (coordinating)1632 GB128 GB

Docker Compose Configurations

Kibana

version: '3'
services:
kibana:
image: docker.elastic.co/kibana/kibana:7.13.1
restart: unless-stopped
hostname: KIB_NODE_NAME
network_mode: bridge
ports:
# HTTP/REST
- 5601:5601/tcp
environment:
TELEMETRY_OPTIN: 'false'
TELEMETRY_ENABLED: 'false'
NEWSFEED_ENABLED: 'false'

SERVER_NAME: 'KIB_NODE_NAME'
SERVER_HOST: '0.0.0.0'
SERVER_PORT: 5601
SERVER_MAXPAYLOADBYTES: 8388608

ELASTICSEARCH_HOSTS: 'https://192.0.2.11:9200'
ELASTICSEARCH_USERNAME: 'kibana_system'
ELASTICSEARCH_PASSWORD: 'CHANGEME'
ELASTICSEARCH_REQUESTTIMEOUT: 132000
ELASTICSEARCH_SHARDTIMEOUT: 120000

#ELASTICSEARCH_SSL_CERTIFICATE: /etc/kibana/certs/node/node.crt
#ELASTICSEARCH_SSL_KEY: /etc/kibana/certs/node/node.key
#ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /etc/kibana/certs/ca/ca.crt
ELASTICSEARCH_SSL_VERIFICATIONMODE: 'none'

KIBANA_AUTOCOMPLETETIMEOUT: 3000
KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000

VIS_TYPE_VEGA_ENABLEEXTERNALURLS: 'true'

XPACK_MAPS_SHOWMAPVISUALIZATIONTYPES: 'true'
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'ElastiFlow_0123456789_0123456789_0123456789'

Coordinating Node for Kibana

version: '3'
services:
es_coord:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: KIB_NODE_NAME
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms8g -Xmx8g'

cluster.name: elastiflow
node.name: KIB_NODE_NAME

node.roles: 'remote_cluster_client'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.11

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Master Node 1

version: '3'
services:
es_master:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_MASTER_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms12g -Xmx12g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_MASTER_1

node.roles: 'master'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.21

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Master Node 2

version: '3'
services:
es_master:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_MASTER_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms12g -Xmx12g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_MASTER_2

node.roles: 'master'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.22

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Master Node 3

version: '3'
services:
es_master:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_MASTER_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms12g -Xmx12g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_MASTER_3

node.roles: 'master'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.23

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Hot Data Node 1

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_1

node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.31

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Hot Data Node 2

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_2

node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.32

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Hot Data Node 3

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_3

node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.33

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Hot Data Node 4

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_4
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_4

node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.34

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Hot Data Node 5

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_5
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_5

node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.35

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Hot Data Node 6

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_6
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_6

node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.36

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Warm Data Node 1

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_1

node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.41

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Warm Data Node 2

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_2

node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.42

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Warm Data Node 3

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_3

node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.43

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Warm Data Node 4

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_4
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_4

node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.44

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Warm Data Node 5

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_5
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_5

node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.45

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Warm Data Node 6

version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_6
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_6

node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.46

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Coordinating Node for the ElastiFlow Collectors

version: '3'
services:
es_coord:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: UFC_NODE_NAME
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms8g -Xmx8g'

cluster.name: elastiflow
node.name: UFC_NODE_NAME

node.roles: 'remote_cluster_client'

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.11

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'