- OS: centos/7
- user: airflow
- http://qiita.com/hatappi/items/0da6cc28e18fa9b5853f
- http://qiita.com/haminiku/items/b431e9cd1cf4e300f8f0
useradd airflow
useradd airflow
yum install -y gcc git zlib-devel bzip2-devel openssl-devel readline-devel sqlite-devel
export PYENV_ROOT=/usr/local/pyenv
curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash
cat > /etc/profile.d/pyenv.sh <<EOH
export PYENV_ROOT=/usr/local/pyenv
export PATH="/usr/local/pyenv/bin:\$PATH"
eval "\$(pyenv init -)"
eval "\$(pyenv virtualenv-init -)"
EOH
source /etc/profile.d/pyenv.sh
pyenv install 3.6.1
pyenv global 3.6.1
cat > /etc/yum.repos.d/MariaDB.repo <<EOH
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/10.1/centos7-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1
EOH
yum clean all
yum install -y MariaDB-server MariaDB-client MariaDB-devel
systemctl enable mariadb
systemctl start mariadb
mysql_secure_installation
mysql -u root -p
> CREATE DATABASE airflow;
> GRANT ALL ON airflow.* TO 'airflow'@'%' IDENTIFIED BY 'airflow';
Root directory
export AIRFLOW_HOME=/var/data/airflow
mkdir -p $AIRFLOW_HOME
chown airflow. $AIRFLOW_HOME
Install airflow
pip install "airflow[mysql, celery]"
Download sample config
curl -L -o /var/data/airflow/airflow.cfg https://raw.githubusercontent.com/apache/incubator-airflow/master/airflow/config_templates/default_airflow.cfg
Edit config
executor = CeleryExecutor
sql_alchemy_conn = mysql://airflow:airflow@localhost:3306/airflow
broker_url = sqla+mysql://airflow:airflow@localhost:3306/airflow
celery_result_backend = db+mysql://airflow:airflow@localhost:3306/airflow
airflow initdb
for dir in /var/log/airflow /var/run/airflow; do
mkdir -p $dir
chown airflow. $dir
done
/usr/lib/systemd/system/airflow-webserver.service
[Unit]
Description=Airflow webserver daemon
After=network.target postgresql.service mysql.service redis.service rabbitmq-server.service
Wants=postgresql.service mysql.service redis.service rabbitmq-server.service
[Service]
Environment="AIRFLOW_HOME=/var/data/airflow"
User=airflow
Group=airflow
Type=simple
ExecStart=/usr/local/pyenv/shims/airflow webserver --port 8080 --pid /var/run/airflow/webserver.pid --stdout /var/log/airflow/webserver.stdout --stderr /var/log/airflow/webserver.stderr --access_logfile /var/log/airflow/access.log --error_logfile /var/log/airflow/error.log
Restart=on-failure
RestartSec=5s
PrivateTmp=true
[Install]
WantedBy=multi-user.target
Web server
systemctl enable airflow-webserver
systemctl start airflow-webserver
Worker
airflow worker