Multi-Tier Cluster
The Multi-Tier Cluster (x-large) deployment is suitable for high ingest rates, while also supporting longer retention periods.
Sizing Parameter | Value |
---|---|
Licensed Units | up to 16 |
Recommended Max. Ingest Rate | 48000 flows/sec |
Retention at Max. Rate | 30 days |
Shards | 6 |
Replicas | 1 |
This multi-tier architecture provides hot data nodes for high-performance ingest, as well as warm nodes for increased storage capacity at a lower cost. Additional warm nodes can be added for even greater retention periods.
Application | CPU Cores | Memory | Storage |
---|---|---|---|
Kibana, Elasticsearch (coordinating) | 8 | 32 GB | 128 GB |
Application | CPU Cores | Memory | Storage |
---|---|---|---|
Elasticsearch (master) | 4 | 32 GB | 128 GB |
Elasticsearch (master) | 4 | 32 GB | 128 GB |
Elasticsearch (master) | 4 | 32 GB | 128 GB |
Application | CPU Cores | Memory | SSD Storage |
---|---|---|---|
Elasticsearch (hot data) | 12-16 | 64 GB | 2 x 4 TB (6.8 TB) |
Elasticsearch (hot data) | 12-16 | 64 GB | 2 x 4 TB (6.8 TB) |
Elasticsearch (hot data) | 12-16 | 64 GB | 2 x 4 TB (6.8 TB) |
Elasticsearch (hot data) | 12-16 | 64 GB | 2 x 4 TB (6.8 TB) |
Elasticsearch (hot data) | 12-16 | 64 GB | 2 x 4 TB (6.8 TB) |
Elasticsearch (hot data) | 12-16 | 64 GB | 2 x 4 TB (6.8 TB) |
Application | CPU Cores | Memory | HDD Storage |
---|---|---|---|
Elasticsearch (warm data) | 8-16 | 64 GB | 4 x 4 TB (13.6 TB) |
Elasticsearch (warm data) | 8-16 | 64 GB | 4 x 4 TB (13.6 TB) |
Elasticsearch (warm data) | 8-16 | 64 GB | 4 x 4 TB (13.6 TB) |
Elasticsearch (warm data) | 8-16 | 64 GB | 4 x 4 TB (13.6 TB) |
Elasticsearch (warm data) | 8-16 | 64 GB | 4 x 4 TB (13.6 TB) |
Elasticsearch (warm data) | 8-16 | 64 GB | 4 x 4 TB (13.6 TB) |
Application | CPU Cores | Memory | Storage |
---|---|---|---|
NetObserv Flow, Elasticsearch (coordinating) | 16 | 32 GB | 128 GB |
Docker Compose Configurations
Kibana
version: '3'
services:
kibana:
image: docker.elastic.co/kibana/kibana:7.13.1
restart: unless-stopped
hostname: KIB_NODE_NAME
network_mode: bridge
ports:
# HTTP/REST
- 5601:5601/tcp
environment:
TELEMETRY_OPTIN: 'false'
TELEMETRY_ENABLED: 'false'
NEWSFEED_ENABLED: 'false'
SERVER_NAME: 'KIB_NODE_NAME'
SERVER_HOST: '0.0.0.0'
SERVER_PORT: 5601
SERVER_MAXPAYLOADBYTES: 8388608
ELASTICSEARCH_HOSTS: 'https://192.0.2.11:9200'
ELASTICSEARCH_USERNAME: 'kibana_system'
ELASTICSEARCH_PASSWORD: 'CHANGEME'
ELASTICSEARCH_REQUESTTIMEOUT: 132000
ELASTICSEARCH_SHARDTIMEOUT: 120000
#ELASTICSEARCH_SSL_CERTIFICATE: /etc/kibana/certs/node/node.crt
#ELASTICSEARCH_SSL_KEY: /etc/kibana/certs/node/node.key
#ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /etc/kibana/certs/ca/ca.crt
ELASTICSEARCH_SSL_VERIFICATIONMODE: 'none'
KIBANA_AUTOCOMPLETETIMEOUT: 3000
KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000
VIS_TYPE_VEGA_ENABLEEXTERNALURLS: 'true'
XPACK_MAPS_SHOWMAPVISUALIZATIONTYPES: 'true'
XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'ElastiFlow_0123456789_0123456789_0123456789'
Coordinating Node for Kibana
version: '3'
services:
es_coord:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: KIB_NODE_NAME
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms8g -Xmx8g'
cluster.name: elastiflow
node.name: KIB_NODE_NAME
node.roles: 'remote_cluster_client'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.11
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Master Node 1
version: '3'
services:
es_master:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_MASTER_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms12g -Xmx12g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_MASTER_1
node.roles: 'master'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.21
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Master Node 2
version: '3'
services:
es_master:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_MASTER_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms12g -Xmx12g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_MASTER_2
node.roles: 'master'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.22
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Master Node 3
version: '3'
services:
es_master:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_MASTER_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms12g -Xmx12g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_MASTER_3
node.roles: 'master'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.23
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Hot Data Node 1
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_1
node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.31
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Hot Data Node 2
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_2
node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.32
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Hot Data Node 3
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_3
node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.33
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Hot Data Node 4
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_4
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_4
node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.34
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Hot Data Node 5
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_5
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_5
node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.35
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Hot Data Node 6
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_HOT_6
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_HOT_6
node.roles: 'data,ingest,transform'
node.attr.temperature: 'hot'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.36
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Warm Data Node 1
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_1
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_1
node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.41
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Warm Data Node 2
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_2
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_2
node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.42
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Warm Data Node 3
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_3
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_3
node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.43
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Warm Data Node 4
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_4
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_4
node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.44
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Warm Data Node 5
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_5
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_5
node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.45
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Elasticsearch Warm Data Node 6
version: '3'
services:
es_data:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: ES_NODE_NAME_WARM_6
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms31g -Xmx31g'
cluster.name: elastiflow
node.name: ES_NODE_NAME_WARM_6
node.roles: 'data'
node.attr.temperature: 'warm'
cluster.routing.allocation.awareness.attributes: 'temperature'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.46
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'
Coordinating Node for NetObserv
version: '3'
services:
es_coord:
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
restart: unless-stopped
hostname: UFC_NODE_NAME
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 131072
hard: 131072
nproc: 8192
fsize: -1
network_mode: bridge
ports:
# HTTP/REST
- 9200:9200/tcp
# Transport
- 9300:9300/tcp
volumes:
# mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
- /var/lib/elasticsearch:/usr/share/elasticsearch/data
- /etc/certs:/usr/share/elasticsearch/config/certificates
environment:
ES_JAVA_OPTS: '-Xms8g -Xmx8g'
cluster.name: elastiflow
node.name: UFC_NODE_NAME
node.roles: 'remote_cluster_client'
bootstrap.memory_lock: 'true'
network.bind_host: 0.0.0.0
network.publish_host: 192.0.2.11
http.port: 9200
http.publish_port: 9200
transport.port: 9300
transport.publish_port: 9300
discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
indices.query.bool.max_clause_count: 8192
search.max_buckets: 250000
action.destructive_requires_name: 'true'
reindex.remote.whitelist: '*:*'
reindex.ssl.verification_mode: 'none'
xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.http.ssl.verification_mode: 'none'
xpack.security.http.ssl.enabled: 'true'
xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
xpack.security.transport.ssl.verification_mode: 'none'
xpack.security.transport.ssl.enabled: 'true'
xpack.monitoring.collection.enabled: 'true'
xpack.monitoring.collection.interval: 30s
xpack.security.enabled: 'true'
xpack.security.audit.enabled: 'false'