Last active
March 26, 2019 23:49
-
-
Save Ardagan/848b89a7a60645203afb1ea77f62c8cc to your computer and use it in GitHub Desktop.
Beam run java worker pipeline
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# build all binaries skipping docker images and running tests | |
./gradlew build -x rat -x javadoc -x test -x website -x docs -x sdist -x lintPy27 -x lintPy27_3 -x lintPy3 -x testWebsite -x spotlessJava -x goTest -x checkstyleMain -x buildDockerImage -x spotbugsMain -x checkstyleTest -x buildLocalWebsite -x goVet -x gofmt | |
# skip compiling java tests | |
-x compileTestJava -x shadowJar -x compileJava | |
# Build java worker jav (DF runner harness) | |
./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:build -x spotlessJava -x rat -x test | |
./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar | |
# Build Python SDK Harness docker image | |
./gradlew -Pdocker-repository-root=gcr.io/dataflow-build/migryz/beam -p sdks/python/container docker | |
gcloud docker -- push gcr.io/dataflow-build/migryz/beam/python:latest | |
# run pipeline | |
virtualenv env | |
. ./env/bin/activate | |
clear && clear && java -cp beam-examples-java-2.12.0-SNAPSHOT.jar org.apache.beam.examples.MinimalWordCount --runner DataflowRunner --num_workers 1 --project google.com:clouddfe --input gs://dataflow-samples/shakespeare/kinglear.txt --output gs://clouddfe-test/staging-$USER/output --temp_location gs://clouddfe-test/staging-$USER/ --worker_harness_container_image gcr.io/dataflow-build/$USER/beam/python:latest --sdk_location build/apache-beam-2.12.0.dev0.tar.gz --dataflow_worker_jar '/usr/local/google/home/migryz/work/osbeam/myfork/runners/google-cloud-dataflow-java/worker/build/libs/beam-runners-google-cloud-dataflow-java-fn-api-worker-2.12.0-SNAPSHOT.jar' | |
clear && clear && python -m apache_beam.examples.wordcount --runner DataflowRunner --num_workers 1 --project google.com:clouddfe --output gs://clouddfe-test/staging-$USER/output --temp_location gs://clouddfe-test/staging-$USER/ --worker_harness_container_image gcr.io/dataflow-build/$USER/beam/python:latest --experiment beam_fn_api --sdk_location build/apache-beam-2.12.0.dev0.tar.gz --dataflow_worker_jar '/usr/local/google/home/migryz/work/osbeam/myfork/runners/google-cloud-dataflow-java/worker/build/libs/beam-runners-google-cloud-dataflow-java-fn-api-worker-2.12.0-SNAPSHOT.jar' | |
# run unittests for python | |
python setup.py nosetests -s --tests apache_beam.runners.portability.fn_api_runner_test:FnApiRunnerTest.test_element_count_metrics | |
# run unittest for java | |
clear && clear && ./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:test --tests org.apache.beam.runners.dataflow.worker.SystemIdToDfePCollectionNameMappingBuilderTest -s --info | |
############################################ | |
# run HelloWorld examples integration test | |
./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:build -x spotlessJava -x rat -x test | |
./gradlew :beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar | |
# Disabling undesired test | |
sed -i 's/^ \/\/ include \"\*\*\/WindowedWordCountIT\.class\"$/ include \"\*\*\/WindowedWordCountIT\.class\"/' ./runners/google-cloud-dataflow-java/examples/build.gradle | |
# Enabling test back | |
sed -i 's/^ include \"\*\*\/WindowedWordCountIT\.class\"$/ \/\/ include \"\*\*\/WindowedWordCountIT\.class\"/' ./runners/google-cloud-dataflow-java/examples/build.gradle | |
clear && clear && ./gradlew :beam-runners-google-cloud-dataflow-java-examples:preCommitFnApiWorker -PdataflowProject=google.com:clouddfe -Pproject=google.com:clouddfe -PgcpProject=google.com:clouddfe -PgcsTempRoot=gs://clouddfe-test/stating-migryz -PdataflowTempRoot=gs://clouddfe-test/staging-migryz -PdockerImageRoot=gcr.io/dataflow-build/migryz/beam | |
clear && clear && ./gradlew :beam-runners-google-cloud-dataflow-java-examples:preCommitLegacyWorker -PdataflowProject=google.com:clouddfe -Pproject=google.com:clouddfe -PgcpProject=google.com:clouddfe -PgcsTempRoot=gs://clouddfe-test/stating-migryz -PdataflowTempRoot=gs://clouddfe-test/staging-migryz -PdockerImageRoot=gcr.io/dataflow-build/migryz/beam | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment