Last active
July 27, 2021 14:58
-
-
Save daniel0x00/3b5107b206efc7418a3fbaa1f387fc44 to your computer and use it in GitHub Desktop.
Converts a PowerShell array object to a Splunk HTTP Event Collector (HEC) valid grouped payload.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
function ConvertTo-SplunkHEC { | |
# PowerShell cmdlet that receives an array and converts it to a Splunk HEC grouped payload. | |
# Author: Daniel Ferreira (@daniel0x00) | |
# License: BSD 3-Clause | |
# Source: https://gist.github.com/daniel0x00/3b5107b206efc7418a3fbaa1f387fc44 | |
<# | |
.SYNOPSIS | |
Receives an array and outputs a Splunk HTTP Event Collector (HEC) valid grouped payload. | |
.EXAMPLE | |
# DNS example: | |
# Note the comma-notation to force pass an array to the pipeline and not a sigle object. | |
,(Resolve-DnsName microsoft.com -DnsOnly) | ConvertTo-SplunkHEC -IndexValue main -SourcetypeValue DNS -SourceValue '"Name":"(?<source>[\w\-\.0-9]+)"' -SourceValueIsRegex -HostValue '(?<host>((([2]([0-4][0-9]|[5][0-5])|[0-1]?[0-9]?[0-9])[.]){3}(([2]([0-4][0-9]|[5][0-5])|[0-1]?[0-9]?[0-9]))))' -HostValueIsRegex -OutputArrayChunks 2 | |
.DESCRIPTION | |
Receives an array input and produces an output array of Splunk HEC payloads. Output JSON Schema: | |
{ | |
"items": { | |
"items": { | |
"properties": { | |
"event": { | |
"properties": {}, | |
"type": "object" | |
}, | |
"host": { | |
"type": "string" | |
}, | |
"index": { | |
"type": "string" | |
}, | |
"source": { | |
"type": "string" | |
}, | |
"sourcetype": { | |
"type": "string" | |
}, | |
"time": { | |
"type": "integer" | |
} | |
}, | |
"required": [ | |
"event", | |
"index", | |
"sourcetype", | |
"source", | |
"host", | |
"time" | |
], | |
"type": "object" | |
}, | |
"type": "array" | |
}, | |
"type": "array" | |
} | |
.PARAMETER InputObject | |
Array. Represents the input object to be converted into Splunk HEC payload. | |
.PARAMETER IndexValue | |
String. Represents the 'index' Splunk field value. | |
.PARAMETER SourcetypeValue | |
String. Represents the 'sourcetype' field value. | |
.PARAMETER SourceValue | |
String. Represents the 'source' field value. | |
Accepts a regular expression string. In this case, it is mandatory to have a regex group called 'source'. | |
.PARAMETER HostValue | |
String. Represents the 'host' field value. | |
Accepts a regular expression string. In this case, it is mandatory to have a regex group called 'host'. | |
.PARAMETER TimeValue | |
String. Represents the '_time' field value. | |
Accepts a regular expression string. In this case, it is mandatory to have a regex group called 'time'. | |
.PARAMETER SourceValueIsRegex | |
Switch. Enable the provided SourceValue string to be a regex instead of plain-text. | |
.PARAMETER HostValueIsRegex | |
Switch. Enable the provided HostValue string to be a regex instead of plain-text. | |
.PARAMETER TimeValueIsRegex | |
Switch. Enable the provided TimeValue string to be a regex instead of plain-text. If this switch is not defined, time will default to UTC time. | |
.PARAMETER OutputArrayChunks | |
Int. Indicates how many objects will be grouped by output chunks. | |
E.g.: if $InputObject contains 5 objects and OutputArrayChunks is configured to 2, ConvertTo-SplunkHEC will produce an output of 3 chunks: 2 of 2 objects each and 1 chunk of 1 object. | |
.EXAMPLE | |
# Requires 'ConvertTo-DotNotation' cmdlet. | |
,(Resolve-DnsName -Name azure.com | Select-Object *,@{n='time';e={Get-Date -Format s}} | Select-Object -ExcludeProperty PSComputerName,RunspaceId,PSShowComputerName -First 1 *, @{n='dotnotation';e={$_ | Select-Object IPAddress, Type, Name, time | ConvertTo-DotNotation}}) | ConvertTo-SplunkHEC -IndexValue dns -SourcetypeValue dns:resolution -SourceValue 'Type=(?<source>[\w\-\.]+)' -SourceValueIsRegex -HostValue 'IPAddress=(?<host>[\w\-\.]+)' -HostValueIsRegex -TimeValue 'time=(?<time>[\w\-:]+)' -TimeValueIsRegex | |
.OUTPUTS | |
[ | |
[ | |
{ | |
"event": { | |
"Address": "23.98.64.158", | |
"CharacterSet": "Unicode", | |
"DataLength": 4, | |
"IP4Address": "23.98.64.158", | |
"IPAddress": "23.98.64.158", | |
"Name": "azure.com", | |
"QueryType": "A", | |
"Section": "Answer", | |
"TTL": 3600, | |
"Type": "A", | |
"dotnotation": [ | |
"IPAddress=23.98.64.158", | |
"Type=A", | |
"Name=azure.com", | |
"time=2021-07-19T16:49:39" | |
], | |
"time": "2021-07-19T16:49:39" | |
}, | |
"host": "23.98.64.158", | |
"index": "dns", | |
"source": "A", | |
"sourcetype": "dns:resolution", | |
"time": 1626706179 | |
}, | |
{ | |
"event": { | |
"Address": "40.74.133.20", | |
"CharacterSet": "Unicode", | |
"DataLength": 4, | |
"IP4Address": "40.74.133.20", | |
"IPAddress": "40.74.133.20", | |
"Name": "azure.com", | |
"QueryType": "A", | |
"Section": "Answer", | |
"TTL": 3600, | |
"Type": "A", | |
"dotnotation": [ | |
"IPAddress=40.74.133.20", | |
"Type=A", | |
"Name=azure.com", | |
"time=2021-07-19T16:49:39" | |
], | |
"time": "2021-07-19T16:49:39" | |
}, | |
"host": "40.74.133.20", | |
"index": "dns", | |
"source": "A", | |
"sourcetype": "dns:resolution", | |
"time": 1626706179 | |
} | |
], | |
[ | |
{ | |
"event": { | |
"Address": "137.135.107.235", | |
"CharacterSet": "Unicode", | |
"DataLength": 4, | |
"IP4Address": "137.135.107.235", | |
"IPAddress": "137.135.107.235", | |
"Name": "azure.com", | |
"QueryType": "A", | |
"Section": "Answer", | |
"TTL": 3600, | |
"Type": "A", | |
"dotnotation": [ | |
"IPAddress=137.135.107.235", | |
"Type=A", | |
"Name=azure.com", | |
"time=2021-07-19T16:49:39" | |
], | |
"time": "2021-07-19T16:49:39" | |
}, | |
"host": "137.135.107.235", | |
"index": "dns", | |
"source": "A", | |
"sourcetype": "dns:resolution", | |
"time": 1626706179 | |
}, | |
{ | |
"event": { | |
"Address": "104.41.9.139", | |
"CharacterSet": "Unicode", | |
"DataLength": 4, | |
"IP4Address": "104.41.9.139", | |
"IPAddress": "104.41.9.139", | |
"Name": "azure.com", | |
"QueryType": "A", | |
"Section": "Answer", | |
"TTL": 3600, | |
"Type": "A", | |
"dotnotation": [ | |
"IPAddress=104.41.9.139", | |
"Type=A", | |
"Name=azure.com", | |
"time=2021-07-19T16:49:39" | |
], | |
"time": "2021-07-19T16:49:39" | |
}, | |
"host": "104.41.9.139", | |
"index": "dns", | |
"source": "A", | |
"sourcetype": "dns:resolution", | |
"time": 1626706179 | |
} | |
] | |
] | |
#> | |
[CmdletBinding()] | |
[OutputType([string])] | |
param( | |
[Parameter(Position=0, Mandatory=$true, ValueFromPipeline=$true)] | |
[System.Array] $InputObject, | |
[Parameter(Mandatory=$true, ValueFromPipeline=$false)] | |
[String] $IndexValue, | |
[Parameter(Mandatory=$true, ValueFromPipeline=$false)] | |
[String] $SourcetypeValue, | |
[Parameter(Mandatory=$true, ValueFromPipeline=$false)] | |
[String] $SourceValue, | |
[Parameter(Mandatory=$true, ValueFromPipeline=$false)] | |
[String] $HostValue, | |
[Parameter(Mandatory=$false, ValueFromPipeline=$false)] | |
[String] $TimeValue, | |
#[Int] $TimeValue = ([int](New-TimeSpan -Start (Get-Date '01/01/1970') -End (Get-Date).ToUniversalTime()).TotalSeconds), | |
[Parameter(Mandatory=$false, ValueFromPipeline=$false)] | |
[Switch] $SourceValueIsRegex = $false, | |
[Parameter(Mandatory=$false, ValueFromPipeline=$false)] | |
[Switch] $HostValueIsRegex = $false, | |
[Parameter(Mandatory=$false, ValueFromPipeline=$false)] | |
[Switch] $TimeValueIsRegex = $false, | |
[Parameter(Mandatory=$false, ValueFromPipeline=$false)] | |
[Int] $OutputArrayChunks = 10, | |
[Parameter(Mandatory=$false, ValueFromPipeline=$false)] | |
[Int] $JSONDepth = 10 | |
) | |
## Output: | |
# Output the array in JSON format sliced in chunks | |
# The intention of this output format is for upstream systems to group, split or distribute the JSON array in the most convenient way. | |
# E.g. On an 10.000 rows input, divide the output into 10 outputs of 1000 rows each and send to Splunk HEC each grouped payload with 5 segs difference in between them. | |
$OutputArray = [System.Collections.Generic.List[Object]]::new() | |
# Group output | |
$counter = [pscustomobject] @{ Value = 0 } | |
# Time value: | |
$Time = ([int](New-TimeSpan -Start (Get-Date '01/01/1970') -End (Get-Date).ToUniversalTime()).TotalSeconds) | |
$GroupedOutput = $InputObject.ForEach({ | |
## Save raw event in JSON to be able to apply regex: | |
$rawEvent = $_ | ConvertTo-JSON -Depth $JSONDepth -Compress | |
$SourceValueOverwritten = $SourceValue | |
$HostValueOverwritten = $HostValue | |
if ($SourceValueIsRegex) { $SourceValueOverwritten = [string](([regex]::Match($rawEvent,$SourceValue)).groups['source'].value) } | |
if ($HostValueIsRegex) { $HostValueOverwritten = [string](([regex]::Match($rawEvent,$HostValue)).groups['host'].value) } | |
if ($TimeValueIsRegex) { $Time = ([int](New-TimeSpan -Start (Get-Date '01/01/1970') -End (Get-Date ([string](([regex]::Match($rawEvent,$TimeValue)).groups['time'].value))).ToUniversalTime() ).TotalSeconds) } | |
[PSCustomObject]@{ | |
event = $_; | |
index = $IndexValue; | |
sourcetype = $SourcetypeValue; | |
source = $SourceValueOverwritten; | |
host = $HostValueOverwritten; | |
time = $Time; | |
} | |
}) | Group-Object -Property { [math]::Floor($counter.Value++ / $OutputArrayChunks) } | |
# Build output array: | |
foreach ($item in $GroupedOutput) { $null = $OutputArray.Add($item.group) } | |
# Outputs the JSON array: | |
$OutputArray | ConvertTo-Json -Depth $JSONDepth -AsArray -Compress | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment