Skip to main content
Version: 6.1

Basic Cluster

The Basic Cluster (medium) deployment is suitable for moderate ingest rates where redundancy is a requirement. It also allows for minimal to no downtime for most maintenance tasks.

Sizing ParameterValue
Licensed Unitsup to 8
Recommended Max. Ingest Rate24000 flows/sec
Retention at Max. Rate10 days
Shards3
Replicas1

Elasticsearch nodes, Kibana and the ElastiFlow Unified Collector are all installed on dedicated systems.

ApplicationCPU CoresMemoryStorage
Kibana416 GB128 GB
ApplicationCPU CoresMemorySSD Storage
Elasticsearch (master/data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (master/data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (master/data)12-1664 GB2 x 4 TB (6.8 TB)
ApplicationCPU CoresMemoryStorage
Flow Collector816 GB128 GB

Docker Compose Configurations

Kibana

version: '3'
services:
kibana:
image: docker.elastic.co/kibana/kibana:7.13.1
restart: unless-stopped
hostname: KIB_NODE_NAME
network_mode: bridge
ports:
# HTTP/REST
- 5601:5601/tcp
environment:
TELEMETRY_OPTIN: 'false'
TELEMETRY_ENABLED: 'false'
NEWSFEED_ENABLED: 'false'

SERVER_NAME: 'KIB_NODE_NAME'
SERVER_HOST: '0.0.0.0'
SERVER_PORT: 5601
SERVER_MAXPAYLOADBYTES: 8388608

ELASTICSEARCH_HOSTS: '["https://192.0.2.11:9200","https://192.0.2.12:9200","https://192.0.2.13:9200"]'
ELASTICSEARCH_USERNAME: 'kibana_system'
ELASTICSEARCH_PASSWORD: 'CHANGEME'
ELASTICSEARCH_REQUESTTIMEOUT: 132000
ELASTICSEARCH_SHARDTIMEOUT: 120000

#ELASTICSEARCH_SSL_CERTIFICATE: /etc/kibana/certs/node/node.crt
#ELASTICSEARCH_SSL_KEY: /etc/kibana/certs/node/node.key
#ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /etc/kibana/certs/ca/ca.crt
ELASTICSEARCH_SSL_VERIFICATIONMODE: 'none'

KIBANA_AUTOCOMPLETETIMEOUT: 3000
KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000

VIS_TYPE_VEGA_ENABLEEXTERNALURLS: 'true'

XPACK_MAPS_SHOWMAPVISUALIZATIONTYPES: 'true'
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'ElastiFlow_0123456789_0123456789_0123456789'

Elasticsearch Node 1

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_1

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.11

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Node 2

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_2

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.12

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Node 3

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_3

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.13

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'