Skip to content

Instantly share code, notes, and snippets.

@bpineau
Created November 13, 2016 23:54
Show Gist options
  • Save bpineau/b0b0a28ca68c8789c27f8b273827f9d2 to your computer and use it in GitHub Desktop.
Save bpineau/b0b0a28ca68c8789c27f8b273827f9d2 to your computer and use it in GitHub Desktop.
/* Terraform setup to evaluate kafka performances on various aws instances types and ebs sizes */
provider "aws" {
region = "eu-west-1"
}
variable "ssh_key_name" {
default = "ben@ici"
}
variable "instances_type" {
default = "t2.medium"
}
variable "data_volume_size" {
default = 500
}
variable "az" {
default = "eu-west-1a"
}
variable "instances_count" {
default = "3"
}
variable "tags" {
default = {
Env_type = "kafka-test"
}
}
data "aws_ami" "centos" {
most_recent = true
filter {
name = "product-code"
values = ["aw0evgkw8e5c1q413zgy5pjce"] # centos 7 64b hvm
}
filter {
name = "root-device-type"
values = ["ebs"]
}
owners = ["aws-marketplace"]
}
data "template_file" "kafka_setup" {
template = <<EOF
sudo curl -jkLH "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u112-b15/jdk-8u112-linux-x64.rpm -o /usr/src/jdk-8u112-linux-x64.rpm
sudo yum -y install /usr/src/jdk-8u112-linux-x64.rpm
sudo curl -L http://apache.mediamirrors.org/kafka/0.10.1.0/kafka_2.11-0.10.1.0.tgz -o /usr/src/kafka_2.11-0.10.1.0.tgz
sudo tar -C /opt -xvzf /usr/src/kafka_2.11-0.10.1.0.tgz
sudo curl -L http://apache.crihan.fr/dist/zookeeper/zookeeper-3.4.9/zookeeper-3.4.9.tar.gz -o /usr/src/zookeeper-3.4.9.tar.gz
sudo tar -C /opt/ -xvzf /usr/src/zookeeper-3.4.9.tar.gz
EOF
}
resource "aws_security_group" "kafka_sg" {
ingress {
protocol = "-1"
from_port = "0"
to_port = "0"
self = true
}
ingress {
from_port = 22
to_port = 22
protocol = "tcp"
cidr_blocks = ["0.0.0.0/0"]
}
egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}
}
resource "aws_instance" "kafka" {
count = "${var.instances_count}"
availability_zone = "${var.az}"
instance_type = "${var.instances_type}"
key_name = "${var.ssh_key_name}"
security_groups = ["${aws_security_group.kafka_sg.name}"]
ami = "${data.aws_ami.centos.id}"
tags = "${merge(var.tags, map("Name", "kafka-instance${count.index}"))}"
associate_public_ip_address = "true"
root_block_device {
volume_size = "20"
delete_on_termination = true
}
connection {
user = "centos"
}
provisioner "remote-exec" {
inline = [
"${data.template_file.kafka_setup.rendered}",
]
}
}
resource "aws_ebs_volume" "kafka_sdb" {
count = "${var.instances_count}"
availability_zone = "${var.az}"
size = "${var.data_volume_size}"
type = "st1"
tags = "${var.tags}"
}
resource "aws_volume_attachment" "ebs_att_sdb_kafka" {
count = "${var.instances_count}"
device_name = "/dev/sdb"
instance_id = "${element(aws_instance.kafka.*.id, count.index)}"
volume_id = "${element(aws_ebs_volume.kafka_sdb.*.id, count.index)}"
}
resource "aws_instance" "client" {
availability_zone = "${var.az}"
instance_type = "${var.instances_type}"
key_name = "${var.ssh_key_name}"
security_groups = ["${aws_security_group.kafka_sg.name}"]
ami = "${data.aws_ami.centos.id}"
tags = "${merge(var.tags, map("Name", "kafka-instance${count.index}"))}"
associate_public_ip_address = "true"
root_block_device {
volume_size = "20"
delete_on_termination = true
}
connection {
user = "centos"
}
provisioner "remote-exec" {
inline = [
"${data.template_file.kafka_setup.rendered}",
]
}
}
resource "null_resource" "config_and_launch" {
count = "${var.instances_count}"
triggers {
cluster_instance_ids = "${join(",", aws_volume_attachment.ebs_att_sdb_kafka.*.id)}"
}
connection {
host = "${element(aws_instance.kafka.*.public_ip, count.index)}"
user = "centos"
}
provisioner "remote-exec" {
inline = [
"printf '${join("\n", aws_instance.kafka.*.private_ip)}' > /tmp/privateips.txt",
"sudo yum -y install vim wget ntp lvm2",
"sudo systemctl start ntpd",
"sudo mkdir -p /var/lib/zookeeper /var/lib/kafka",
"sudo parted --script --align optimal /dev/xvdb -- mklabel gpt mkpart primary 0% 100% set 1 lvm on",
"sudo pvcreate /dev/xvdb1",
"sudo vgcreate vg01 /dev/xvdb1",
"sudo lvcreate -l 100%FREE -n lv01 vg01",
"sudo mkfs.ext4 /dev/vg01/lv01",
"sudo mount /dev/vg01/lv01 /var/lib/kafka",
"sudo rm -rf /var/lib/kafka/lost*",
"sudo cp /opt/zookeeper-3.4.9/conf/zoo_sample.cfg /opt/zookeeper-3.4.9/conf/zoo.cfg",
"sudo sed -i -e 's#^dataDir=.*#dataDir=/var/lib/zookeeper#' /opt/zookeeper-3.4.9/conf/zoo.cfg",
"awk '{ C += 1 ; print \"server.\" C \"=\" $1 \":2182:2183\" }' < /tmp/privateips.txt | sudo tee -a /opt/zookeeper-3.4.9/conf/zoo.cfg",
"grep -n \"^$(hostname -i)$\" /tmp/privateips.txt |cut -d: -f1 | sudo tee /var/lib/zookeeper/myid",
"sudo sed -i -e 's/^broker.id=.*/broker.id=-1/' /opt/kafka_2.11-0.10.1.0/config/server.properties",
"sudo sed -i -e 's#^log.dirs=.*#log.dirs=/var/lib/kafka/#' /opt/kafka_2.11-0.10.1.0/config/server.properties",
"sudo sed -i -e \"s#^zookeeper.connect=.*#zookeeper.connect=$(sed -e 's/$/:2181/' /tmp/privateips.txt | paste -sd ',')#\" /opt/kafka_2.11-0.10.1.0/config/server.properties",
"cd / && sudo env JVMFLAGS='-Xmx512M -Xms512M' /opt/zookeeper-3.4.9/bin/zkServer.sh start",
"cd / && sleep 10 && sudo env KAFKA_HEAP_OPTS='-Xmx1024M -Xms1024M' /opt/kafka_2.11-0.10.1.0/bin/kafka-server-start.sh /opt/kafka_2.11-0.10.1.0/config/server.properties > /tmp/kafka.logs 2>&1 &",
]
}
}
output "private_ips" {
value = ["${aws_instance.kafka.*.private_ip}"]
}
output "public_ips" {
value = ["${aws_instance.kafka.*.public_ip}"]
}
output "client_ip" {
value = "${aws_instance.client.public_ip}"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment