Created
December 7, 2023 01:56
-
-
Save pflooky/07266512a1c8d338dfe7609fd45dab39 to your computer and use it in GitHub Desktop.
Example of Data Caterer connecting to OpenMetadata to generate and validate data in JSON and CSV file. Customised field values for JSON file as well.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class AdvancedOpenMetadataSourcePlanRun extends PlanRun { | |
val openMetadataSource = metadataSource.openMetadata( | |
"http://host.docker.internal:8585/api", | |
OPEN_METADATA_AUTH_TYPE_OPEN_METADATA, | |
Map( | |
OPEN_METADATA_JWT_TOKEN -> "abc123", //find under settings/bots/ingestion-bot/token | |
OPEN_METADATA_TABLE_FQN -> "sample_data.ecommerce_db.shopify.raw_customer" | |
) | |
) | |
val jsonTask = json("my_json", "/opt/app/data/json") | |
.schema(openMetadataSource) | |
.schema( | |
field.name("platform").oneOf("website", "mobile"), | |
field.name("customer").schema(field.name("sex").oneOf("M", "F", "O")) | |
) | |
val csvTask = csv("my_csv", "/opt/app/data/csv").schema(openMetadataSource) | |
val conf = configuration.enableGeneratePlanAndTasks(true).enableGenerateValidations(true) | |
execute(conf, jsonTask) | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment