Evan Galpin egalpin
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import static org.junit.Assert.assertTrue; | |
| import org.apache.beam.sdk.testing.NeedsRunner; | |
| import org.apache.beam.sdk.testing.PAssert; | |
| import org.apache.beam.sdk.transforms.Count; | |
| import org.apache.beam.sdk.transforms.Create; | |
| import org.apache.beam.sdk.transforms.MapElements; | |
| import org.apache.beam.sdk.transforms.PeriodicImpulse; | |
| import org.apache.beam.sdk.transforms.SerializableFunction; | |
| import org.apache.beam.sdk.transforms.Values; |
egalpin
/ JsonMutationFn.java
Last active
February 23, 2024 00:51
JsonMutationFn using Beam and Jackson Databind
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import com.fasterxml.jackson.core.JsonPointer; | |
| import com.fasterxml.jackson.databind.json.JsonMapper; | |
| import com.fasterxml.jackson.databind.node.ObjectNode; | |
| import java.util.ArrayList; | |
| import java.util.List; | |
| import org.apache.beam.sdk.transforms.DoFn; | |
| import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting; | |
| /** | |
| * This DoFn is mostly used to rename json fields, but is flexible enough, thanks to json pointers, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument; | |
| import com.google.auto.value.AutoValue; | |
| import java.io.Serializable; | |
| import java.util.concurrent.TimeUnit; | |
| import org.apache.beam.sdk.coders.Coder; | |
| import org.apache.beam.sdk.coders.KvCoder; | |
| import org.apache.beam.sdk.state.BagState; | |
| import org.apache.beam.sdk.state.CombiningState; | |
| import org.apache.beam.sdk.state.State; |
egalpin
/ DatastoreEntitySerializer.java
Last active
June 9, 2022 17:01
Jackson JSON serializer for Google Datastore Entity
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import com.fasterxml.jackson.core.JsonGenerator; | |
| import com.fasterxml.jackson.databind.SerializerProvider; | |
| import com.fasterxml.jackson.databind.ser.std.StdSerializer; | |
| import com.google.cloud.Timestamp; | |
| import com.google.cloud.datastore.Blob; | |
| import com.google.cloud.datastore.Entity; | |
| import com.google.cloud.datastore.Key; | |
| import com.google.cloud.datastore.Value; | |
| import java.io.IOException; | |
| import java.io.UnsupportedEncodingException; |
egalpin
/ minimal_heterogeneous_repro.py
Created
September 8, 2021 17:10
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import argparse | |
| from typing import Iterable | |
| from typing import List | |
| from typing import Tuple | |
| from typing import Union | |
| import apache_beam as beam | |
| from apache_beam.transforms.combiners import ToDictCombineFn | |
| from apache_beam.transforms.combiners import ToSetCombineFn |
egalpin
/ merge_junit_results.py
Created
September 28, 2016 13:41
— forked from cgoldberg/merge_junit_results.py
Merge multiple JUnit XML results files into a single results file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/env python | |
| # | |
| # Corey Goldberg, Dec 2012 | |
| # | |
| import os | |
| import sys | |
| import xml.etree.ElementTree as ET |