Last active
December 11, 2017 16:36
-
-
Save sc68cal/26089ff59073586ac91aa1c50b06ecc0 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
From b536eb89d895cb80489ac970923e389a0d73636e Mon Sep 17 00:00:00 2001 | |
From: "Sean M. Collins" <sean@coreitpro.com> | |
Date: Mon, 11 Dec 2017 10:47:15 -0500 | |
Subject: [PATCH] Evict wtp-k8s-3 from the cluster | |
Hardware issues with node 3 means we'll promote node 4 to a master | |
--- | |
k8s-envs/vpcbeta-production/hosts | 12 +++++++----- | |
1 file changed, 7 insertions(+), 5 deletions(-) | |
diff --git a/k8s-envs/vpcbeta-production/hosts b/k8s-envs/vpcbeta-production/hosts | |
index 01faddb..15492f4 100644 | |
--- a/k8s-envs/vpcbeta-production/hosts | |
+++ b/k8s-envs/vpcbeta-production/hosts | |
@@ -1,18 +1,20 @@ | |
wtp-k8s-1.projwrigley.com ansible_host=10.171.203.112 ip=10.171.203.112 ansible_user=root database_node_ord=1 | |
wtp-k8s-2.projwrigley.com ansible_host=10.171.203.193 ip=10.171.203.193 ansible_user=root database_node_ord=2 | |
-wtp-k8s-3.projwrigley.com ansible_host=10.171.201.20 ip=10.171.201.20 ansible_user=root database_node_ord=3 | |
-wtp-k8s-4.projwrigley.com ansible_host=10.171.201.249 ip=10.171.201.249 ansible_user=root | |
+wtp-k8s-4.projwrigley.com ansible_host=10.171.201.249 ip=10.171.201.249 ansible_user=root database_node_ord=3 | |
wtp-k8s-5.projwrigley.com ansible_host=10.171.201.112 ip=10.171.201.112 ansible_user=root | |
wtp-k8s-6.projwrigley.com ansible_host=10.171.201.66 ip=10.171.201.66 ansible_user=root | |
[kube-master] | |
-wtp-k8s-[1:3].projwrigley.com | |
+wtp-k8s-[1:2].projwrigley.com | |
+wtp-k8s-4.projwrigley.com | |
[etcd] | |
-wtp-k8s-[1:3].projwrigley.com | |
+wtp-k8s-[1:2].projwrigley.com | |
+wtp-k8s-4.projwrigley.com | |
[kube-node] | |
-wtp-k8s-[1:6].projwrigley.com | |
+wtp-k8s-[1:2].projwrigley.com | |
+wtp-k8s-[4:6].projwrigley.com | |
[database:children] | |
kube-master |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
2017-12-11 16:02:03.146978 | TASK [etcd : Configure | Check if cluster is healthy] ************************** | |
2017-12-11 16:02:03.147120 | Monday 11 December 2017 16:02:03 +0000 (0:00:02.783) 0:01:06.562 ******* | |
2017-12-11 16:02:13.490291 | ok: [wtp-k8s-1.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 cluster-health | grep -q 'cluster is healthy'", "delta": "0:00:10.048627", "end": "2017-12-11 10:02:13.471955", "failed": false, "rc": 0, "start": "2017-12-11 10:02:03.423328", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} | |
2017-12-11 16:02:13.503112 | ok: [wtp-k8s-2.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 cluster-health | grep -q 'cluster is healthy'", "delta": "0:00:10.047946", "end": "2017-12-11 10:02:13.485331", "failed": false, "rc": 0, "start": "2017-12-11 10:02:03.437385", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} | |
2017-12-11 16:02:13.520347 | ok: [wtp-k8s-4.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 cluster-health | grep -q 'cluster is healthy'", "delta": "0:00:10.049270", "end": "2017-12-11 10:02:13.503088", "failed": false, "rc": 0, "start": "2017-12-11 10:02:03.453818", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} | |
2017-12-11 16:02:13.529611 | | |
2017-12-11 16:02:13.529663 | TASK [etcd : Configure | Check if member is in cluster] ************************ | |
2017-12-11 16:02:13.529811 | Monday 11 December 2017 16:02:13 +0000 (0:00:10.382) 0:01:16.944 ******* | |
2017-12-11 16:02:13.855650 | ok: [wtp-k8s-1.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --no-sync --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 member list | grep -q 10.171.203.112", "delta": "0:00:00.033135", "end": "2017-12-11 10:02:13.839410", "failed": false, "rc": 0, "start": "2017-12-11 10:02:13.806275", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} | |
2017-12-11 16:02:13.875416 | ok: [wtp-k8s-2.projwrigley.com] => {"changed": false, "cmd": "/usr/local/bin/etcdctl --no-sync --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 member list | grep -q 10.171.203.193", "delta": "0:00:00.033358", "end": "2017-12-11 10:02:13.859151", "failed": false, "rc": 0, "start": "2017-12-11 10:02:13.825793", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} | |
2017-12-11 16:02:13.902197 | fatal: [wtp-k8s-4.projwrigley.com]: FAILED! => {"changed": false, "cmd": "/usr/local/bin/etcdctl --no-sync --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 member list | grep -q 10.171.201.249", "delta": "0:00:00.033663", "end": "2017-12-11 10:02:13.887385", "failed": true, "msg": "non-zero return code", "rc": 1, "start": "2017-12-11 10:02:13.853722", "stderr": "", "stderr_lines": [], "stdout": "", "stdout_lines": []} | |
2017-12-11 16:02:13.902343 | ...ignoring | |
2017-12-11 16:02:13.911672 | | |
2017-12-11 16:02:13.911739 | TASK [etcd : Configure | Add member to the cluster if it is not there] ********* | |
2017-12-11 16:02:13.911885 | Monday 11 December 2017 16:02:13 +0000 (0:00:00.382) 0:01:17.326 ******* | |
2017-12-11 16:02:16.293740 | fatal: [wtp-k8s-4.projwrigley.com]: FAILED! => {"changed": true, "cmd": "/usr/local/bin/etcdctl --peers=https://10.171.203.112:2379,https://10.171.203.193:2379,https://10.171.201.249:2379 member add etcd3 https://10.171.201.249:2380", "delta": "0:00:02.042013", "end": "2017-12-11 10:02:16.273793", "failed": true, "msg": "non-zero return code", "rc": 1, "start": "2017-12-11 10:02:14.231780", "stderr": "client: etcd cluster is unavailable or misconfigured; error #0: client: etcd member https://10.171.203.193:2379 has no leader\n; error #1: client: etcd member https://10.171.203.112:2379 has no leader\n; error #2: client: endpoint https://10.171.201.20:2379 exceeded header timeout", "stderr_lines": ["client: etcd cluster is unavailable or misconfigured; error #0: client: etcd member https://10.171.203.193:2379 has no leader", "; error #1: client: etcd member https://10.171.203.112:2379 has no leader", "; error #2: client: endpoint https://10.171.201.20:2379 exceeded header timeout"], "stdout": "", "stdout_lines": []} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment