Created
January 31, 2019 14:29
-
-
Save guyboertje/2b1134cbbfed46eca7fb5598812f099e to your computer and use it in GitHub Desktop.
Logstash xml file processing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?xml version="1.0" encoding="UTF-8" ?> | |
<ExecutionLogModel> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="initializing..." shapetype="initializing..." shapelabel="" shapeextendedinfo=""> | |
<Message>Executing Process Performance Management- Past(Child)-STAGE (Continuation f_0_0)</Message> | |
</LogEvent> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape10" shapetype="Try/Catch" shapelabel="" shapeextendedinfo=""> | |
<Message>Executing Try/Catch Shape continuation as f_0_0 with 1 document(s).</Message> | |
</LogEvent> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape10" shapetype="Try/Catch" shapelabel="" shapeextendedinfo=""> | |
<Message>Shape executed successfully in 82 ms.</Message> | |
</LogEvent> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape13" shapetype="Map" shapelabel="" shapeextendedinfo="Performance Management- Past Year(groovy to FF)"> | |
<Message>Executing Map with 1 document(s).</Message> | |
</LogEvent> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape13" shapetype="Map" shapelabel="" shapeextendedinfo="Performance Management- Past Year(groovy to FF)"> | |
<Message>Shape executed successfully in 2 ms.</Message> | |
</LogEvent> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape14" shapetype="Return Documents" shapelabel="Success" shapeextendedinfo=""> | |
<Message>Adding 1 document(s) to Return Documents return store.</Message> | |
</LogEvent> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape14" shapetype="Return Documents" shapelabel="Success" shapeextendedinfo=""> | |
<Message>Shape executed successfully in 1 ms.</Message> | |
</LogEvent> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="cleanup..." shapetype="cleanup..." shapelabel="" shapeextendedinfo=""> | |
<Message>Returning 1 document(s) in Return Documents return store.</Message> | |
</LogEvent> | |
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="cleanup..." shapetype="cleanup..." shapelabel="" shapeextendedinfo=""> | |
<Message>Process execution completed normally.</Message> | |
</LogEvent> | |
</ExecutionLogModel> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"shapelabel" => "", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "initializing...", | |
"Message" => "Executing Process Performance Management- Past(Child)-STAGE (Continuation f_0_0)", | |
"level" => "INFO", | |
"shapeextendedinfo" => "", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "initializing...", | |
"host" => "Elastics-MacBook-Pro.local" | |
} | |
{ | |
"shapelabel" => "", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "shape10", | |
"Message" => "Executing Try/Catch Shape continuation as f_0_0 with 1 document(s).", | |
"level" => "INFO", | |
"shapeextendedinfo" => "", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "Try/Catch", | |
"host" => "Elastics-MacBook-Pro.local" | |
} | |
{ | |
"shapelabel" => "", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "shape10", | |
"Message" => "Shape executed successfully in 82 ms.", | |
"level" => "INFO", | |
"shapeextendedinfo" => "", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "Try/Catch", | |
"host" => "Elastics-MacBook-Pro.local" | |
} | |
{ | |
"shapelabel" => "", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "shape13", | |
"Message" => "Executing Map with 1 document(s).", | |
"level" => "INFO", | |
"shapeextendedinfo" => "Performance Management- Past Year(groovy to FF)", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "Map", | |
"host" => "Elastics-MacBook-Pro.local" | |
} | |
{ | |
"shapelabel" => "", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "shape13", | |
"Message" => "Shape executed successfully in 2 ms.", | |
"level" => "INFO", | |
"shapeextendedinfo" => "Performance Management- Past Year(groovy to FF)", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "Map", | |
"host" => "Elastics-MacBook-Pro.local" | |
} | |
{ | |
"shapelabel" => "Success", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "shape14", | |
"Message" => "Adding 1 document(s) to Return Documents return store.", | |
"level" => "INFO", | |
"shapeextendedinfo" => "", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "Return Documents", | |
"host" => "Elastics-MacBook-Pro.local" | |
} | |
{ | |
"shapelabel" => "Success", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "shape14", | |
"Message" => "Shape executed successfully in 1 ms.", | |
"level" => "INFO", | |
"shapeextendedinfo" => "", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "Return Documents", | |
"host" => "Elastics-MacBook-Pro.local" | |
} | |
{ | |
"shapelabel" => "", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "cleanup...", | |
"Message" => "Returning 1 document(s) in Return Documents return store.", | |
"level" => "INFO", | |
"shapeextendedinfo" => "", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "cleanup...", | |
"host" => "Elastics-MacBook-Pro.local" | |
} | |
{ | |
"shapelabel" => "", | |
"@timestamp" => 2019-01-31T14:25:40.653Z, | |
"@version" => "1", | |
"time" => "2018-07-10T04:12:26Z", | |
"shapename" => "cleanup...", | |
"Message" => "Process execution completed normally.", | |
"level" => "INFO", | |
"shapeextendedinfo" => "", | |
"path" => "/elastic/tmp/testing/logs/logs_sample.xml", | |
"shapetype" => "cleanup...", | |
"host" => "Elastics-MacBook-Pro.local" | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
input { | |
file { | |
sincedb_path => "/elastic/tmp/testing/confs/test-jk-xmlpretty.sincedb" | |
path => "/elastic/tmp/testing/logs/logs_sample.xml" | |
mode => "read" | |
file_completed_action => "log" | |
file_completed_log_path => "/elastic/tmp/testing/confs/test-jk-xmlpretty-done.txt" | |
delimiter => "øhåyø" | |
} | |
} | |
filter { | |
xml { | |
remove_namespaces => true | |
store_xml => true | |
target => "[log_event]" | |
source => "[message]" | |
force_array => false | |
force_content => false | |
} | |
split { | |
field => "[log_event][LogEvent]" | |
remove_field => ["message"] | |
} | |
ruby { | |
code => ' | |
event.remove("[log_event]")["LogEvent"].each{|k,v| event.set(k,v)} | |
' | |
} | |
} | |
output { stdout { codec => "rubydebug"} } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment