ELK Installation Steps

HAProxy Settings in /etc/openstack_deploy/user_variables.yml

elasticsearch_http_port: 9200
elasticsearch_tcp_port: 9300

haproxy_extra_services:
  - service:
      haproxy_service_name: elasticsearch-http
      haproxy_backend_nodes: "{{ groups['elasticsearch_all'] | default([]) }}"
      haproxy_ssl: "{{ haproxy_ssl }}"
      haproxy_port: "{{ elasticsearch_http_port }}"
      haproxy_balance_type: http
      haproxy_backend_options:
        - "forwardfor"
        - "httpchk"
        - "httplog"
  - service:
      haproxy_service_name: elasticsearch-tcp
      haproxy_backend_nodes: "{{ groups['elasticsearch_all'] | default([]) }}"
      haproxy_port: "{{ elasticsearch_tcp_port }}"
      haproxy_balance_type: tcp
  - service:
      haproxy_service_name: kibana
      haproxy_ssl: False
      haproxy_backend_nodes: "{{ groups['kibana_all'] | default([]) }}"
      haproxy_port: 8443
      haproxy_balance_type: tcp
      haproxy_backend_options:
        - "ssl-hello-chk"

Update ansible-role-requirements.yml

- name: ansible-elasticsearch
  scm: git
  src: https://github.com/elastic/ansible-elasticsearch
  version: master
- name: rpc-role-logstash
  scm: git
  src: https://github.com/rcbops/rpc-role-logstash
  version: master
- name: rpc-role-filebeat
  scm: git
  src: https://github.com/d34dh0r53/rpc-role-filebeat
  version: master
- name: rpc-role-kibana
  scm: git
  src: https://github.com/d34dh0r53/rpc-role-kibana
  version: master

Update user_secrets.yml

kibana_password:

# /opt/openstack-ansible/scripts/pw-token-gen --file /etc/openstack_deploy/user_secrets.yml

Playbook files elk-elasticsearch.yml

---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

- name: Setup ElasticSearch host
  hosts: elasticsearch_all
  environment:
    ES_SKIP_SET_KERNEL_PARAMETERS: "true"
  pre_tasks:
    - name: Create ElasticSearch data directory on host
      file:
        path: "/openstack/{{ container_name }}/var/lib/elasticsearch"
        state: directory
        group: "root"
        owner: "root"
        mode:  "0755"
        recurse: no
      delegate_to: "{{ physical_host }}"
      when: not (is_metal | bool)
    - name: ElasticSearch extra lxc config
      lxc_container:
        name: "{{ container_name }}"
        container_command: |
          [[ ! -d "/var/lib/elasticsearch" ]] && mkdir -p "/var/lib/elasticsearch"
        container_config:
          - "lxc.mount.entry=/openstack/{{ container_name }}/var/lib/elasticsearch var/lib/elasticsearch none bind 0 0"
          - "lxc.aa_profile=unconfined"
      delegate_to: "{{ physical_host }}"
      when: not (is_metal | bool)
      tags:
        - elasticsearch-pre-install
    - name: Flush net cache
      command: /usr/local/bin/lxc-system-manage flush-net-cache
      delegate_to: "{{ physical_host }}"
      when: not (is_metal | bool)
      tags:
        - elasticsearch-pre-install
    - name: Wait for container ssh
      wait_for:
        port: "22"
        delay: 5
        host: "{{ container_address }}"
      delegate_to: "{{ physical_host }}"
      when: not (is_metal | bool)
      tags:
        - elasticsearch-pre-install
  roles:
    - { role: "ansible-elasticsearch",
        es_instance_name: "openstack",
        es_data_dirs: "/var/lib/elasticsearch",
        es_log_dir: "/var/log/elasticsearch",
    es_config: {
        node.name: "{{ container_name }}",
        cluster.name: "openstack",
        http.port: 9200,
        transport.tcp.port: 9300,
        node.data: true,
        node.master: true,
        bootstrap.memory_lock: true,
        network.host: "{{ container_address }}"
        }
    }
  vars:
    is_metal: "{{ properties.is_metal | default(False) }}"
    es_scripts: false
    es_templates: false
    es_version_lock: false
    es_heap_size: 1g
    es_api_host: "{{ container_address }}"
    es_api_port: 9200
    es_version: "5.4.0"
    es_major_version: "5.x"

elk-logstash.yml

---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

- name: Setup Logstash host
  hosts: logstash_all
  environment:
    ES_SKIP_SET_KERNEL_PARAMETERS: "true"
  roles:
    - role: "rpc-role-logstash"

elk-filebeat.yml

---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

- name: Setup Filebeat log shiping
  hosts: all
  max_fail_percentage: 20
  roles:
    - role: "rpc-role-filebeat"
      tags: filebeat-install

elk-kibana.yml

---
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

- name: Setup Kibana host
  hosts: kibana_all
  roles:
    - role: "rpc-role-kibana"

/etc/openstack_deploy/env.d files elasticsearch.yml

---
component_skel:
  elasticsearch:
    belongs_to:
      - elasticsearch_all

container_skel:
  elasticsearch_container:
    belongs_to:
      - log_containers
    contains:
      - elasticsearch
    properties:
      service_name: elasticsearch

logstash.yml

---
component_skel:
  logstash:
    belongs_to:
      - logstash_all

container_skel:
  logstash_container:
    belongs_to:
      - log_containers
    contains:
      - logstash
    properties:
      service_name: logstash

kibana.yml

---
component_skel:
  kibana:
    belongs_to:
      - kibana_all

container_skel:
  kibana_container:
    belongs_to:
      - log_containers
    contains:
      - kibana
    properties:
      service_name: kibana

/etc/openstack_deploy/conf.d/log_hosts.yml modifications

log_hosts:
  logging1:
    ip: 10.29.236.110
    container_vars:
      openstack_user_kernel_options:
        - { key: 'vm.max_map_count', value: 262144 }
        - { key: 'vm.mmap_min_addr', value: 65536 }
        - { key: 'kernel.kptr_restrict', value: 1 }
        - { key: 'net.ipv4.tcp_syncookies', value: 1 }
        - { key: 'kernel.sysrq', value: 176 }
        - { key: 'fs.protected_hardlinks', value: 1 }
        - { key: 'fs.protected_symlinks', value: 1 }
        - { key: 'kernel.yama.ptrace_scope', value: 1 }
        - { key: 'kernel.printk', value: '4 4 1 7' }

d34dh0r53/elk-docs-draft.md