Created
March 23, 2016 13:28
-
-
Save yomexzo/b25793d5c9d14872c160 to your computer and use it in GitHub Desktop.
Troublshooting tools
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/env python | |
import re | |
from os import path, listdir | |
from os.path import isfile, join | |
config_path = path.abspath(path.expanduser('~') + '/.ssh/config') | |
template = 'Host {}\n\tStrictHostKeyChecking no\n\tHostName {}\n\tUser {}\n\n' | |
begin = '#WRW HOST VARS - START' | |
end = '#WRW HOST VARS - END' | |
def readHosts(hosts_dir) : | |
hosts = [] | |
for filename in listdir(hosts_dir): | |
if filename == '.DS_Store': | |
continue | |
file = open(join(hosts_dir, filename), 'r') | |
lines = file.read() | |
pattern = re.compile('^ansible_ssh_user:(.+?)$|^ansible_ssh_host:(.+?)$', flags=re.MULTILINE) | |
matches = pattern.findall(lines) | |
hostname = filename | |
ipaddress = matches[0][1].strip() | |
username = matches[1][0].strip() | |
hosts.append((hostname, ipaddress, username)) | |
file.close() | |
return hosts | |
def writeConfig(hosts): | |
global template, config_path, begin, end | |
config = open(config_path, 'r+') | |
# Fetch existing content | |
content = config.read() | |
match = re.search(begin + '(.*)' + end, content, re.DOTALL) | |
if match: | |
content = content.replace(match.group(0), '') | |
config.close() | |
# append new content | |
content += begin | |
content += '\n\n' | |
for host in hosts: | |
content += template.format(host[0], host[1], host[2]) | |
content += end | |
# Write final content | |
config = open(config_path, 'w') | |
config.write(content) | |
config.close() | |
print 'Hosts added successfully' | |
hosts_dir1 = './inventory/scrapers/host_vars' | |
hosts1 = readHosts(hosts_dir1) | |
hosts_dir2 = './inventory/proxies/host_vars' | |
hosts2 = readHosts(hosts_dir2) | |
hosts = hosts1 + hosts2 | |
writeConfig(hosts) | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
printf "Which servers are we troubleshooting today?\n" | |
read servers | |
printf "\nOkay, I'm going to start dealing with them one by one\n" | |
ssh_timeout=5 | |
servers_provision=() | |
servers_error=() | |
servers_check_proxies=() | |
join() { | |
# $1 is return variable name | |
# $2 is sep | |
# $3... are the elements to join | |
local retname=$1 sep=$2 ret=$3 | |
shift 3 || shift $(($#)) | |
printf -v "$retname" "%s" "$ret${@/#/$sep}" | |
} | |
for server in $servers; do | |
status=$(tugboat info $server -a status) | |
echo "Treating '$server'" | |
if [[ "$status" =~ "off" ]]; then | |
echo "Server is off" | |
elif [[ "$status" =~ "active" ]]; then | |
supervisor_info=`(ssh -o ConnectTimeout=$ssh_timeout $server 'supervisorctl status') 2>&1`; | |
if [[ "$supervisor_info" =~ "unix:///var/run/supervisor.sock" ]]; then | |
x=$(ssh $server 'supervisord -c /etc/supervisor/supervisord.conf') | |
echo "Supervisor reloaded - sock" | |
elif [[ "$supervisor_info" =~ "nucleus_scraper.py:nucleus_proc0 RUNNING" ]]; then | |
echo "Nucleus is running. Check proxies. I'll provision anyways." | |
servers_check_proxies+=("$server") | |
servers_provision+=("$server") | |
elif [[ "$supervisor_info" =~ "nucleus_scraper.py:nucleus_proc0" ]]; then | |
echo "Nucleus is NOT running. I'll provision." | |
servers_provision+=("$server") | |
elif [[ "$supervisor_info" =~ "Connection timed out during banner exchange" ]]; then | |
echo "Rebuilding this server" | |
x=$(tugboat rebuild $server ubuntu-14-04-x64 --confirm -q) | |
# tugboat rebuild $server ubuntu-14-04-x64 --confirm | |
servers_provision+=("$server") | |
else | |
echo "I don't know what to do with this one - $supervisor_info" | |
servers_error+=("$server") | |
fi | |
else | |
echo "Error getting info from digitalocean" | |
fi | |
printf "\n" | |
done | |
if [[ ${#servers_provision[@]} > 0 ]]; then | |
echo "I'll wait for 60s in order to allow rebuilt servers to complete before provisioning." | |
sleep 60 | |
join sss "," "${servers_provision[@]}" | |
echo "ansible-playbook -i inventory/scrapers/hosts nucleus-provision.yml --ask-vault-pass --private-key ~/.ssh/wrw-default.pem --limit $sss" | |
ansible-playbook -i inventory/scrapers/hosts nucleus-provision.yml --ask-vault-pass --private-key ~/.ssh/wrw-default.pem --limit $sss | |
fi | |
if [[ ${#servers_error[@]} > 0 ]]; then | |
join IFS "," "${servers_error[@]}" | |
echo "The following servers have one issue or the other - $IFS" | |
fi | |
if [[ ${#servers_check_proxies[@]} > 0 ]]; then | |
join sss "," "${servers_check_proxies[@]}" | |
echo "You should check the proxies for the following servers - $sss" | |
fi | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment