Skip to main content

Basic Cluster

The Basic Cluster (medium) deployment is suitable for moderate ingest rates where redundancy is a requirement. It also allows for minimal to no downtime for most maintenance tasks.

Sizing ParameterValue
Licensed Coresup to 8
Recommended Max. Ingest Rate24000 flows/sec
Retention at Max. Rate10 days
Shards3
Replicas1

Elasticsearch nodes, Kibana and the ElastiFlow Unified Flow Collector are all installed on dedicated systems.

ApplicationCPU CoresMemoryStorage
Kibana416 GB128 GB
ApplicationCPU CoresMemorySSD Storage
Elasticsearch (master/data)12-1696-128 GB2 x 4 TB (6.8 TB)
Elasticsearch (master/data)12-1696-128 GB2 x 4 TB (6.8 TB)
Elasticsearch (master/data)12-1696-128 GB2 x 4 TB (6.8 TB)
ApplicationCPU CoresMemoryStorage
Flow Collector816 GB128 GB

Docker Compose Configurations#

Kibana#

version: '3'
services:
kibana:
image: docker.elastic.co/kibana/kibana:7.13.1
restart: unless-stopped
hostname: KIB_NODE_NAME
network_mode: bridge
ports:
# HTTP/REST
- 5601:5601/tcp
environment:
TELEMETRY_OPTIN: 'false'
TELEMETRY_ENABLED: 'false'
NEWSFEED_ENABLED: 'false'
SERVER_NAME: 'KIB_NODE_NAME'
SERVER_HOST: '0.0.0.0'
SERVER_PORT: 5601
SERVER_MAXPAYLOADBYTES: 8388608
ELASTICSEARCH_HOSTS: '["https://192.0.2.11:9200","https://192.0.2.12:9200","https://192.0.2.13:9200"]'
ELASTICSEARCH_USERNAME: 'kibana_system'
ELASTICSEARCH_PASSWORD: 'CHANGEME'
ELASTICSEARCH_REQUESTTIMEOUT: 132000
ELASTICSEARCH_SHARDTIMEOUT: 120000
#ELASTICSEARCH_SSL_CERTIFICATE: /etc/kibana/certs/node/node.crt
#ELASTICSEARCH_SSL_KEY: /etc/kibana/certs/node/node.key
#ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /etc/kibana/certs/ca/ca.crt
ELASTICSEARCH_SSL_VERIFICATIONMODE: 'none'
KIBANA_AUTOCOMPLETETIMEOUT: 3000
KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000
VIS_TYPE_VEGA_ENABLEEXTERNALURLS: 'true'
XPACK_MAPS_SHOWMAPVISUALIZATIONTYPES: 'true'
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'ElastiFlow_0123456789_0123456789_0123456789'

Elasticsearch Node 1#

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_1
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.11
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Node 2#

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_2
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.12
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Node 3#

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_3
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.13
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'