Created
January 12, 2020 13:53
-
-
Save gxercavins/17f1ce31e9960e5482dfd3502d762cb2 to your computer and use it in GitHub Desktop.
SO question 59697878
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import argparse | |
import logging | |
import apache_beam as beam | |
from apache_beam.options.pipeline_options import SetupOptions | |
from apache_beam.options.pipeline_options import PipelineOptions | |
def prediction_handler(element): | |
logging.info("Hi") | |
yield element | |
def main(argv=None): | |
parser = argparse.ArgumentParser() | |
known_args, pipeline_args = parser.parse_known_args(argv) | |
pipeline_options = PipelineOptions(pipeline_args) | |
pipeline_options.view_as(SetupOptions).save_main_session = True | |
p = beam.Pipeline(options=pipeline_options) | |
output_tweets = (p | |
| 'start' >> beam.Create(['Start']) | |
| 'predict sentiment' >> beam.FlatMap( | |
lambda messages: prediction_handler(messages)) | |
) | |
result = p.run() | |
result.wait_until_finish() | |
if __name__ == '__main__': | |
logging.getLogger().setLevel(logging.INFO) | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment