Last active
December 19, 2017 18:38
-
-
Save stefanobaghino/50b2919eef6acb108d3647ccf570d937 to your computer and use it in GitHub Desktop.
Error while running a dockerized Airflow that uses Mesos as a distributed executor - FIXED
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
FROM python:2.7-jessie | |
LABEL maintainer="Teralytics AG" | |
LABEL version=1 | |
# Mesos-related image build-time arguments | |
ARG MESOS_VERSION=1.0.1 | |
ARG MESOS_VERSION_QUALIFIER=2.0.93.debian81 | |
# Airflow-related image build-time arguments | |
ARG AIRFLOW_VERSION=1.8.1 | |
ARG AIRFLOW_PKG=apache-airflow | |
ARG AIRFLOW_EXTRAS=crypto:jdbc:hdfs:hive:postgres:slack | |
# Install Mesos and Airflow dependencies | |
RUN echo "deb http://repos.mesosphere.io/debian/ jessie main" > /etc/apt/sources.list.d/mesosphere.list && \ | |
apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF && \ | |
apt-get -y update && \ | |
apt-get -y install python-pandas mesos="$MESOS_VERSION"-"$MESOS_VERSION_QUALIFIER" && \ | |
apt-get clean && rm -rf /var/lib/apt/lists/* && \ | |
cp /usr/lib/python2.7/site-packages/mesos* -r /usr/local/lib/python2.7/site-packages/ | |
# Install Airflow | |
RUN pip install "$AIRFLOW_PKG"=="$AIRFLOW_VERSION" | |
# Install extra package for Airflow | |
RUN echo "$AIRFLOW_EXTRAS" | tr ':' '\n' | xargs -i@ pip install "$AIRFLOW_PKG"[@]=="$AIRFLOW_VERSION" | |
# Default command for the container: run both the scheduler and the webserver | |
ENTRYPOINT airflow initdb && airflow scheduler & airflow webserver |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Traceback (most recent call last): | |
File "/usr/local/bin/airflow", line 17, in <module> | |
from airflow import configuration | |
File "/usr/local/lib/python2.7/site-packages/airflow/__init__.py", line 31, in <module> | |
from airflow.models import DAG | |
File "/usr/local/lib/python2.7/site-packages/airflow/models.py", line 63, in <module> | |
from airflow.executors import DEFAULT_EXECUTOR, LocalExecutor | |
File "/usr/local/lib/python2.7/site-packages/airflow/executors/__init__.py", line 47, in <module> | |
from airflow.contrib.executors.mesos_executor import MesosExecutor | |
File "/usr/local/lib/python2.7/site-packages/airflow/contrib/executors/mesos_executor.py", line 23, in <module> | |
import mesos.native | |
ImportError: No module named native | |
Traceback (most recent call last): | |
File "/usr/local/bin/airflow", line 17, in <module> | |
from airflow import configuration | |
File "/usr/local/lib/python2.7/site-packages/airflow/__init__.py", line 31, in <module> | |
from airflow.models import DAG | |
File "/usr/local/lib/python2.7/site-packages/airflow/models.py", line 63, in <module> | |
from airflow.executors import DEFAULT_EXECUTOR, LocalExecutor | |
File "/usr/local/lib/python2.7/site-packages/airflow/executors/__init__.py", line 47, in <module> | |
from airflow.contrib.executors.mesos_executor import MesosExecutor | |
File "/usr/local/lib/python2.7/site-packages/airflow/contrib/executors/mesos_executor.py", line 23, in <module> | |
import mesos.native | |
ImportError: No module named native |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hey stefan, what are you achieving by doing:
cp /usr/lib/python2.7/site-packages/mesos* -r /usr/local/lib/python2.7/site-packages/