- Running inside an unconfined lxc container
- ES 2.x worked fine
- Seems to be related to binding to interface
- System info:
uname
# uname -a
Linux logging1-elasticsearch-container-4701b6ca 4.4.0-75-generic #96-Ubuntu SMP Thu Apr 20 09:56:33 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
dpkg
# dpkg -l | grep elastic
ii elasticsearch 5.3.2 all Elasticsearch is a distributed RESTful search engine built for the cloud. Reference documentation can be found at https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html and the 'Elasticsearch: The Definitive Guide' book can be found at https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html
- Contents of
/etc/elasticsearch/
# find /etc/elasticsearch -ls
1860151 4 drwxr-x--- 3 root elasticsearch 4096 May 2 10:15 /etc/elasticsearch
1864874 4 -rw-r--r-- 1 elasticsearch elasticsearch 46 May 2 10:05 /etc/elasticsearch/jvm.options
1860056 4 -rw-r--r-- 1 elasticsearch elasticsearch 140 May 2 10:11 /etc/elasticsearch/elasticsearch.yml
1864875 4 drwxr-x--- 2 elasticsearch elasticsearch 4096 Apr 24 11:18 /etc/elasticsearch/scripts
1864873 4 -rw-r--r-- 1 elasticsearch elasticsearch 610 May 2 08:55 /etc/elasticsearch/log4j2.properties
- Processes
# ps auxwww
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 1 0.0 0.2 37160 5460 ? Ss Apr28 0:02 /sbin/init
root 40 0.0 0.4 43852 9680 ? Ss Apr28 0:01 /lib/systemd/systemd-journald
syslog 73 0.0 0.1 256400 2940 ? Ssl Apr28 0:00 /usr/sbin/rsyslogd -n
root 74 0.0 0.1 28980 2584 ? Ss Apr28 0:00 /usr/sbin/cron -f
root 144 0.0 0.1 16128 2968 ? Ss Apr28 0:00 /sbin/dhclient -1 -v -pf /run/dhclient.eth0.pid -lf /var/lib/dhcp/dhclient.eth0.leases -I -df /var/lib/dhcp/dhclient6.eth0.leases eth0
root 193 0.0 0.2 65520 5940 ? Ss Apr28 0:00 /usr/sbin/sshd -D
root 195 0.0 0.0 15756 2004 pts/3 Ss+ Apr28 0:00 /sbin/agetty --noclear --keep-baud pts/3 115200 38400 9600 vt220
root 196 0.0 0.0 15756 1888 lxc/console Ss+ Apr28 0:00 /sbin/agetty --noclear --keep-baud console 115200 38400 9600 vt220
root 197 0.0 0.0 15756 1996 pts/0 Ss+ Apr28 0:00 /sbin/agetty --noclear --keep-baud pts/0 115200 38400 9600 vt220
root 198 0.0 0.1 15756 2064 pts/2 Ss+ Apr28 0:00 /sbin/agetty --noclear --keep-baud pts/2 115200 38400 9600 vt220
root 199 0.0 0.0 15756 1972 pts/1 Ss+ Apr28 0:00 /sbin/agetty --noclear --keep-baud pts/1 115200 38400 9600 vt220
root 16493 0.0 0.1 37364 3248 pts/3 R+ 10:45 0:00 ps auxwww
root 18156 0.0 0.5 17496 11120 ? Ssl Apr29 1:22 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat
root 24658 0.0 0.1 21328 3824 pts/3 Ss 09:37 0:00 /bin/bash
root 24687 0.0 0.2 38352 4608 pts/3 S 09:40 0:00 journalctl -f
root 24809 0.0 0.1 21228 3640 ? Ss 09:54 0:00 /bin/bash
root 25092 0.0 0.1 14768 3372 ? S+ 09:58 0:13 watch -d -n 1 ss -lntp
- Networking information
# ip -o a l
1: lo inet 127.0.0.1/8 scope host lo\ valid_lft forever preferred_lft forever
1: lo inet6 ::1/128 scope host \ valid_lft forever preferred_lft forever
95: eth0 inet 10.0.3.45/24 brd 10.0.3.255 scope global eth0\ valid_lft forever preferred_lft forever
95: eth0 inet6 fe80::216:3eff:feca:2ebf/64 scope link \ valid_lft forever preferred_lft forever
97: eth1 inet 10.29.239.123/22 brd 10.29.239.255 scope global eth1\ valid_lft forever preferred_lft forever
97: eth1 inet6 fe80::216:3eff:fee6:8449/64 scope link \ valid_lft forever preferred_lft forever
# ip route
default via 10.0.3.1 dev eth0
10.0.3.0/24 dev eth0 proto kernel scope link src 10.0.3.45
10.29.236.0/22 dev eth1 proto kernel scope link src 10.29.239.123
# ss -lneutp
Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port
udp UNCONN 0 0 *:68 *:* users:(("dhclient",pid=144,fd=6)) ino:597149 sk:1f <->
tcp LISTEN 0 128 *:22 *:* users:(("sshd",pid=193,fd=3)) ino:596636 sk:1 <->
tcp LISTEN 0 128 :::22 :::* users:(("sshd",pid=193,fd=4)) ino:596645 sk:2 v6only:1 <->
- Config info:
/etc/elasticsearch/log4j2.properties
# cat log4j2.properties
appender.rolling.type = RollingFile
appender.rolling.name = rolling
appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
appender.rolling.layout.type = PatternLayout
appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %.10000m%n
appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}.log
appender.rolling.policies.type = Policies
appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
appender.rolling.policies.time.interval = 1
appender.rolling.policies.time.modulate = true/etc/elasticsearch/elasticsearch.yml
# cat elasticsearch.yml
cluster.name: openstack
node.name: "Testing-123"
network.host: _global_
path:
logs: /var/log/elasticsearch
data: /var/lib/elasticsearchjvm.options
# cat jvm.options
-Xms1024m
-Xmx1024m
-Dlog4j2.disable.jmx=true
- Log from failed start
# service elasticsearch start
May 02 10:33:52 logging1-elasticsearch-container-4701b6ca systemd[1]: Failed to reset devices.list on /system.slice/elasticsearch.service: Operation not permitted
May 02 10:33:52 logging1-elasticsearch-container-4701b6ca systemd[1]: Starting Elasticsearch...
May 02 10:33:52 logging1-elasticsearch-container-4701b6ca systemd[1]: Started Elasticsearch.
May 02 10:33:58 logging1-elasticsearch-container-4701b6ca elasticsearch[14068]: Exception: java.security.AccessControlException thrown from the UncaughtExceptionHandler in thread "Thread-4"
May 02 10:33:58 logging1-elasticsearch-container-4701b6ca systemd[1]: elasticsearch.service: Main process exited, code=exited, status=1/FAILURE
May 02 10:33:58 logging1-elasticsearch-container-4701b6ca systemd[1]: elasticsearch.service: Unit entered failed state.
May 02 10:33:58 logging1-elasticsearch-container-4701b6ca systemd[1]: elasticsearch.service: Failed with result 'exit-code'.
- Log and
sscommand once we comment outnetwork.hostand restart
# vi elasticsearch.yml
root@logging1-elasticsearch-container-4701b6ca:/etc/elasticsearch# service elasticsearch start
May 02 10:37:16 logging1-elasticsearch-container-4701b6ca systemd[1]: Failed to reset devices.list on /system.slice/elasticsearch.service: Operation not permitted
May 02 10:37:16 logging1-elasticsearch-container-4701b6ca systemd[1]: Starting Elasticsearch...
May 02 10:37:16 logging1-elasticsearch-container-4701b6ca systemd[1]: Started Elasticsearch.
root@logging1-elasticsearch-container-4701b6ca:/etc/elasticsearch# ss -lneutp
Netid State Recv-Q Send-Q Local Address:Port Peer Address:Port
udp UNCONN 0 0 *:68 *:* users:(("dhclient",pid=144,fd=6)) ino:597149 sk:1f <->
tcp LISTEN 0 128 *:22 *:* users:(("sshd",pid=193,fd=3)) ino:596636 sk:1 <->
tcp LISTEN 0 128 ::ffff:127.0.0.1:9200 :::* users:(("java",pid=14759,fd=136)) uid:107 ino:1031348 sk:22 v6only:0 <->
tcp LISTEN 0 128 ::1:9200 :::* users:(("java",pid=14759,fd=134)) uid:107 ino:1031347 sk:23 v6only:1 <->
tcp LISTEN 0 128 ::ffff:127.0.0.1:9300 :::* users:(("java",pid=14759,fd=121)) uid:107 ino:1032245 sk:20 v6only:0 <->
tcp LISTEN 0 128 ::1:9300 :::* users:(("java",pid=14759,fd=119)) uid:107 ino:1032236 sk:21 v6only:1 <->
tcp LISTEN 0 128 :::22 :::* users:(("sshd",pid=193,fd=4)) ino:596645 sk:2 v6only:1 <->
# curl localhost:9200
{
"name" : "Testing-123",
"cluster_name" : "openstack",
"cluster_uuid" : "57rtf02ERqGcVxTvdNPm-w",
"version" : {
"number" : "5.3.2",
"build_hash" : "3068195",
"build_date" : "2017-04-24T16:15:59.481Z",
"build_snapshot" : false,
"lucene_version" : "6.4.2"
},
"tagline" : "You Know, for Search"
}
network.hostoptions tried:
_global__site__eth0__eth1_10.0.3.45127.0.0.1,10.0.3.45["127.0.0.1", "10.0.3.45"]
network.host:
- "127.0.0.1"
- "10.0.3.45"
network.host:
"127.0.0.1"
"10.0.3.45"
All result with the same failure mode, /var/log/elasticsearch/openstack.log is created but empty.