Version: 6.4

Advanced Cluster

The Advanced Cluster (large) deployment is suitable for high ingest rates and is easily expanded as necessary.

Sizing Parameter	Value
Licensed Units	up to 16
Recommended Max. Ingest Rate	48000 flows/sec
Retention at Max. Rate	10 days
Shards	6
Replicas	1

Dedicated Master nodes as well as Coordinating nodes are leveraged for greater flexibility and performance. Additional data nodes can be added for even higher ingest rates.

Application	CPU Cores	Memory	Storage
Kibana, Elasticsearch (coordinating)	8	32 GB	128 GB

Application	CPU Cores	Memory	Storage
Elasticsearch (master)	4	32 GB	128 GB
Elasticsearch (master)	4	32 GB	128 GB
Elasticsearch (master)	4	32 GB	128 GB

Application	CPU Cores	Memory	Storage
Elasticsearch (data)	12-16	64 GB	2 x 4 TB (6.8 TB)
Elasticsearch (data)	12-16	64 GB	2 x 4 TB (6.8 TB)
Elasticsearch (data)	12-16	64 GB	2 x 4 TB (6.8 TB)
Elasticsearch (data)	12-16	64 GB	2 x 4 TB (6.8 TB)
Elasticsearch (data)	12-16	64 GB	2 x 4 TB (6.8 TB)
Elasticsearch (data)	12-16	64 GB	2 x 4 TB (6.8 TB)

Application	CPU Cores	Memory	Storage
Flow Collector, Elasticsearch (coordinating)	16	32 GB	128 GB

Docker Compose Configurations

Kibana

version: '3'
services:
  kibana:
    image: docker.elastic.co/kibana/kibana:7.13.1
    restart: unless-stopped
    hostname: KIB_NODE_NAME
    network_mode: bridge
    ports:
      # HTTP/REST
      - 5601:5601/tcp
    environment:
      TELEMETRY_OPTIN: 'false'
      TELEMETRY_ENABLED: 'false'
      NEWSFEED_ENABLED: 'false'

      SERVER_NAME: 'KIB_NODE_NAME'
      SERVER_HOST: '0.0.0.0'
      SERVER_PORT: 5601
      SERVER_MAXPAYLOADBYTES: 8388608

      ELASTICSEARCH_HOSTS: 'https://192.0.2.11:9200'
      ELASTICSEARCH_USERNAME: 'kibana_system'
      ELASTICSEARCH_PASSWORD: 'CHANGEME'
      ELASTICSEARCH_REQUESTTIMEOUT: 132000
      ELASTICSEARCH_SHARDTIMEOUT: 120000

      #ELASTICSEARCH_SSL_CERTIFICATE: /etc/kibana/certs/node/node.crt
      #ELASTICSEARCH_SSL_KEY: /etc/kibana/certs/node/node.key
      #ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES: /etc/kibana/certs/ca/ca.crt
      ELASTICSEARCH_SSL_VERIFICATIONMODE: 'none'

      KIBANA_AUTOCOMPLETETIMEOUT: 3000
      KIBANA_AUTOCOMPLETETERMINATEAFTER: 2500000

      VIS_TYPE_VEGA_ENABLEEXTERNALURLS: 'true'

      XPACK_MAPS_SHOWMAPVISUALIZATIONTYPES: 'true'
      XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY: 'ElastiFlow_0123456789_0123456789_0123456789'

Coordinating Node for Kibana

version: '3'
services:
  es_coord:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: KIB_NODE_NAME
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms8g -Xmx8g'

      cluster.name: elastiflow
      node.name: KIB_NODE_NAME

      node.roles: 'remote_cluster_client'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.11

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Master Node 1

version: '3'
services:
  es_master:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_MASTER_1
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms12g -Xmx12g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_MASTER_1

      node.roles: 'master'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.21

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
      cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Master Node 2

version: '3'
services:
  es_master:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_MASTER_2
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms12g -Xmx12g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_MASTER_2

      node.roles: 'master'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.22

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
      cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Master Node 3

version: '3'
services:
  es_master:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_MASTER_3
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms12g -Xmx12g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_MASTER_3

      node.roles: 'master'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.23

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'
      cluster.initial_master_nodes: 'ES_NODE_NAME_MASTER_1,ES_NODE_NAME_MASTER_2,ES_NODE_NAME_MASTER_3'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Data Node 1

version: '3'
services:
  es_data:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_DATA_1
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_DATA_1

      node.roles: 'data,ingest,transform'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.31

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Data Node 2

version: '3'
services:
  es_data:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_DATA_2
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_DATA_2

      node.roles: 'data,ingest,transform'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.32

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Data Node 3

version: '3'
services:
  es_data:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_DATA_3
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_DATA_3

      node.roles: 'data,ingest,transform'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.33

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Data Node 4

version: '3'
services:
  es_data:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_DATA_4
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_DATA_4

      node.roles: 'data,ingest,transform'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.34

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Data Node 5

version: '3'
services:
  es_data:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_DATA_5
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_DATA_5

      node.roles: 'data,ingest,transform'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.35

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Elasticsearch Data Node 6

version: '3'
services:
  es_data:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: ES_NODE_NAME_DATA_6
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms31g -Xmx31g'

      cluster.name: elastiflow
      node.name: ES_NODE_NAME_DATA_6

      node.roles: 'data,ingest,transform'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.36

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Coordinating Node for the ElastiFlow Collectors

version: '3'
services:
  es_coord:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.13.1
    restart: unless-stopped
    hostname: UFC_NODE_NAME
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 131072
        hard: 131072
      nproc: 8192
      fsize: -1
    network_mode: bridge
    ports:
      # HTTP/REST
      - 9200:9200/tcp
      # Transport
      - 9300:9300/tcp
    volumes:
      # mkdir /var/lib/elasticsearch && chown -R 1000:1000 /var/lib/elasticsearch
      - /var/lib/elasticsearch:/usr/share/elasticsearch/data
      - /etc/certs:/usr/share/elasticsearch/config/certificates
    environment:
      ES_JAVA_OPTS: '-Xms8g -Xmx8g'

      cluster.name: elastiflow
      node.name: UFC_NODE_NAME

      node.roles: 'remote_cluster_client'

      bootstrap.memory_lock: 'true'

      network.bind_host: 0.0.0.0
      network.publish_host: 192.0.2.11

      http.port: 9200
      http.publish_port: 9200

      transport.port: 9300
      transport.publish_port: 9300

      discovery.seed_hosts: '192.0.2.21,192.0.2.22,192.0.2.23'

      indices.query.bool.max_clause_count: 8192
      search.max_buckets: 250000

      action.destructive_requires_name: 'true'

      reindex.remote.whitelist: '*:*'
      reindex.ssl.verification_mode: 'none'

      xpack.security.http.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.http.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.http.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.http.ssl.verification_mode: 'none'
      xpack.security.http.ssl.enabled: 'true'

      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/certificates/node/node.key
      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/certificates/node/node.crt
      xpack.security.transport.ssl.certificate_authorities: /usr/share/elasticsearch/config/certificates/ca/ca.crt
      xpack.security.transport.ssl.verification_mode: 'none'
      xpack.security.transport.ssl.enabled: 'true'

      xpack.monitoring.collection.enabled: 'true'
      xpack.monitoring.collection.interval: 30s

      xpack.security.enabled: 'true'
      xpack.security.audit.enabled: 'false'

Docker Compose Configurations​

Kibana​

Coordinating Node for Kibana​

Elasticsearch Master Node 1​

Elasticsearch Master Node 2​

Elasticsearch Master Node 3​

Elasticsearch Data Node 1​

Elasticsearch Data Node 2​

Elasticsearch Data Node 3​

Elasticsearch Data Node 4​

Elasticsearch Data Node 5​

Elasticsearch Data Node 6​

Coordinating Node for the ElastiFlow Collectors​