Skip to main content
Version: 7.4

Basic Cluster

The Basic Cluster (medium) deployment is suitable for moderate ingest rates where redundancy is a requirement. It also allows for minimal to no downtime for most maintenance tasks.

Sizing ParameterValue
Licensed Unitsup to 8
Recommended Max. Ingest Rate24000 flows/sec
Retention at Max. Rate10 days
Shards3
Replicas1

Elasticsearch nodes, Kibana and NetObserv are all installed on dedicated systems.

ApplicationCPU CoresMemoryStorage
Kibana416 GB128 GB
ApplicationCPU CoresMemorySSD Storage
Elasticsearch (master/data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (master/data)12-1664 GB2 x 4 TB (6.8 TB)
Elasticsearch (master/data)12-1664 GB2 x 4 TB (6.8 TB)
ApplicationCPU CoresMemoryStorage
NetObserv Flow816 GB128 GB

Docker Compose Configurations

Kibana

version: '3'
services:
kibana:
image: docker.elastic.co/kibana/kibana:7.13.1
restart: unless-stopped
hostname: KIB_NODE_NAME
network_mode: bridge
ports:
# HTTP/REST
- 5601:5601/tcp
environment:
TELEMETRY_OPTIN: 'false'
TELEMETRY_ENABLED: 'false'
NEWSFEED_ENABLED: 'false'

SERVER_NAME: 'KIB_NODE_NAME'
SERVER_HOST: '0.0.0.0'
SERVER_PORT: 5601
SERVER_MAXPAYLOADBYTES: 8388608

ELASTICSEARCH_HOSTS: '["https://192.0.2.11:9200","https://192.0.2.12:9200","https://192.0.2.13:9200"]'
ELASTICSEARCH_USERNAME: 'kibana_system'
ELASTICSEARCH_PASSWORD: 'CHANGEME'
ELASTICSEARCH_REQUESTTIMEOUT: 132000
ELASTICSEARCH_SHARDTIMEOUT: 120000

#ELASTICSEARCH_SSL_CERTIFICATE: /etc/kibana/certs/node/node.crt
#ELASTICSEARCH_SSL_KEY: /etc/kibana/certs/node/node.key
#ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /etc/kibana/certs/ca/ca.crt
ELASTICSEARCH_SSL_VERIFICATIONMODE: 'none'

KIBANA_AUTOCOMPLETETIMEOUT: 3000
KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000

VIS_TYPE_VEGA_ENABLEEXTERNALURLS: 'true'

XPACK_MAPS_SHOWMAPVISUALIZATIONTYPES: 'true'
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'ElastiFlow_0123456789_0123456789_0123456789'

Elasticsearch Node 1

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_1

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.11

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Node 2

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_2

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.12

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'

Elasticsearch Node 3

version: '3'
services:
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
container_name: elasticsearch
restart: unless-stopped
hostname: ES_NODE_NAME_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'

cluster.name: elastiflow
node.name: ES_NODE_NAME_3

bootstrap.memory_lock: 'true'

network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.13

http.port: 9200
http.publish_port: 9200

transport.port: 9300
transport.publish_port: 9300

discovery.seed_hosts: '192.0.2.11,192.0.2.12,192.0.2.13'
cluster.initial_master_nodes: 'ES_NODE_NAME_1,ES_NODE_NAME_2,ES_NODE_NAME_3'

indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000

action.destructive_requires_name: 'true'

reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'

xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'

xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'

xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s

xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'