Created
January 22, 2026 21:20
-
-
Save nathanmcnulty/d6160033db2a44fa01f7ac16e4caf069 to your computer and use it in GitHub Desktop.
Get-XdrEndpointDeviceTimeline.ps1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| function Get-XdrEndpointDeviceTimeline { | |
| <# | |
| .SYNOPSIS | |
| Retrieves the timeline of events for a specific device from Microsoft Defender XDR. | |
| .DESCRIPTION | |
| Gets the timeline of security events for a device from the Microsoft Defender XDR portal with options to filter by date range and other parameters. | |
| Uses parallel chunked requests (1-hour intervals) to improve performance and support longer date ranges up to 180 days. | |
| .PARAMETER DeviceId | |
| The unique identifier of the device. Accepts pipeline input and can also be specified as MachineId. Use this parameter set when identifying the device by ID. | |
| .PARAMETER MachineDnsName | |
| The DNS name of the machine. Use this parameter set when identifying the device by DNS name. | |
| .PARAMETER MarkedEventsOnly | |
| Only return events that have been marked in the timeline. | |
| .PARAMETER SenseClientVersion | |
| Optional. The version of the Sense client. | |
| .PARAMETER GenerateIdentityEvents | |
| Whether to generate identity events. Defaults to $true. | |
| .PARAMETER IncludeIdentityEvents | |
| Whether to include identity events. Defaults to $true. | |
| .PARAMETER SupportMdiOnlyEvents | |
| Whether to support MDI-only events. Defaults to $true. | |
| .PARAMETER FromDate | |
| The start date for the timeline. Defaults to 1 hour before current time. | |
| .PARAMETER ToDate | |
| The end date for the timeline. Defaults to current time. | |
| .PARAMETER LastNDays | |
| Specifies the number of days to look back. Overrides FromDate and ToDate if specified. | |
| .PARAMETER DoNotUseCache | |
| Whether to bypass the cache. Defaults to $false. | |
| .PARAMETER ForceUseCache | |
| Whether to force using the cache. Defaults to $false. | |
| .PARAMETER PageSize | |
| The number of events to return per page. Defaults to 200. | |
| .PARAMETER IncludeSentinelEvents | |
| Whether to include Sentinel events. Defaults to $false. | |
| .PARAMETER ThrottleLimit | |
| The maximum number of concurrent requests. Defaults to 10. | |
| .PARAMETER OutputPath | |
| Optional. The path to store temporary JSON files. Defaults to a temp folder. | |
| .PARAMETER KeepTempFiles | |
| If specified, keeps the temporary JSON files after merging. | |
| .EXAMPLE | |
| Get-XdrEndpointDeviceTimeline -DeviceId "2bec169acc9def3ebd0bf8cdcbd9d16eb37e50e2" | |
| Retrieves the last hour of timeline events for the specified device. | |
| .EXAMPLE | |
| Get-XdrEndpointDeviceTimeline -MachineDnsName "computer.contoso.com" | |
| Retrieves the last hour of timeline events using the machine DNS name. | |
| .EXAMPLE | |
| Get-XdrEndpointDeviceTimeline -DeviceId "2bec169acc9def3ebd0bf8cdcbd9d16eb37e50e2" -FromDate (Get-Date).AddDays(-7) -ToDate (Get-Date) | |
| Retrieves timeline events for the last 7 days using parallel requests. | |
| .EXAMPLE | |
| Get-XdrEndpointDeviceTimeline -DeviceId "2bec169acc9def3ebd0bf8cdcbd9d16eb37e50e2" -LastNDays 90 -ThrottleLimit 5 | |
| Retrieves 90 days of timeline events with 5 concurrent requests. | |
| .EXAMPLE | |
| "2bec169acc9def3ebd0bf8cdcbd9d16eb37e50e2" | Get-XdrEndpointDeviceTimeline | |
| Retrieves timeline events using pipeline input. | |
| #> | |
| [OutputType([System.Object[]])] | |
| [CmdletBinding(DefaultParameterSetName = 'ByDeviceId')] | |
| param ( | |
| [Parameter(Mandatory, ValueFromPipeline, ValueFromPipelineByPropertyName, ParameterSetName = 'ByDeviceId')] | |
| [Alias('MachineId')] | |
| [string]$DeviceId, | |
| [Parameter(Mandatory, ParameterSetName = 'ByMachineDnsName')] | |
| [string]$MachineDnsName, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [datetime]$FromDate = ((Get-Date).AddHours(-1)), | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [datetime]$ToDate = (Get-Date), | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [int]$LastNDays, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [ValidateRange(1, 1000)] | |
| [int]$PageSize = 200, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [switch]$MarkedEventsOnly, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [string]$SenseClientVersion, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [bool]$GenerateIdentityEvents = $true, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [bool]$IncludeIdentityEvents = $true, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [bool]$SupportMdiOnlyEvents = $true, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [bool]$DoNotUseCache = $false, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [bool]$ForceUseCache = $false, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [bool]$IncludeSentinelEvents = $false, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [ValidateRange(1, 20)] | |
| [int]$ThrottleLimit = 10, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [string]$OutputPath, | |
| [Parameter(ParameterSetName = 'ByDeviceId')] | |
| [Parameter(ParameterSetName = 'ByMachineDnsName')] | |
| [switch]$KeepTempFiles | |
| ) | |
| begin { | |
| Update-XdrConnectionSettings | |
| } | |
| process { | |
| if ($PSBoundParameters.ContainsKey('LastNDays')) { | |
| $ToDate = Get-Date | |
| $FromDate = $ToDate.AddDays(-$LastNDays) | |
| } | |
| # New limit is 180 days since we chunk into 1-day requests | |
| if (($ToDate - $FromDate).TotalDays -gt 180) { | |
| throw "The time range between FromDate and ToDate cannot exceed 180 days." | |
| } | |
| # Determine the device identifier to use in the URI | |
| $deviceIdentifier = if ($PSCmdlet.ParameterSetName -eq 'ByDeviceId') { $DeviceId } else { (Get-XdrEndpointDevice -MachineSearchPrefix $MachineDnsName).MachineId } | |
| # Get the ComputerDnsName for folder naming - handle null/multiple results gracefully | |
| $device = Get-XdrEndpointDevice -MachineSearchPrefix $deviceIdentifier | Select-Object -First 1 | |
| $computerDnsName = if ($device -and $device.ComputerDnsName) { $device.ComputerDnsName } else { $deviceIdentifier } | |
| # Sanitize folder name - ensure we have a valid value | |
| if ([string]::IsNullOrWhiteSpace($computerDnsName)) { | |
| $computerDnsName = $deviceIdentifier | |
| } | |
| $safeFolderName = $computerDnsName -replace '[\\/:*?"<>|]', '_' | |
| # Set up output directory | |
| $baseTempPath = if ($OutputPath) { $OutputPath } else { "$env:TEMP\XdrTimeline" } | |
| $deviceTempPath = "$baseTempPath\$safeFolderName" | |
| $runId = [guid]::NewGuid().ToString('N').Substring(0, 8) | |
| $runTempPath = "$deviceTempPath\$runId" | |
| if (-not (Test-Path $runTempPath)) { | |
| New-Item -Path $runTempPath -ItemType Directory -Force | Out-Null | |
| } | |
| Write-Verbose "Temporary files will be stored in: $runTempPath" | |
| # Build the base query parameters (without date range) | |
| $baseQueryParams = @{ | |
| GenerateIdentityEvents = $GenerateIdentityEvents | |
| IncludeIdentityEvents = $IncludeIdentityEvents | |
| SupportMdiOnlyEvents = $SupportMdiOnlyEvents | |
| DoNotUseCache = $DoNotUseCache | |
| ForceUseCache = $ForceUseCache | |
| PageSize = $PageSize | |
| IncludeSentinelEvents = $IncludeSentinelEvents | |
| MarkedEventsOnly = $MarkedEventsOnly.IsPresent | |
| SenseClientVersion = $SenseClientVersion | |
| MachineDnsName = if ($PSBoundParameters.ContainsKey('MachineDnsName')) { $MachineDnsName } else { $null } | |
| } | |
| # Generate date chunks - always use 1-hour chunks for optimal parallelism | |
| $dateChunks = [System.Collections.Generic.List[hashtable]]::new() | |
| $totalDays = ($ToDate - $FromDate).TotalDays | |
| $totalHours = $totalDays * 24 | |
| # Use 1-hour chunks for maximum parallelism and performance | |
| $currentDate = $FromDate | |
| $chunkIndex = 0 | |
| while ($currentDate -lt $ToDate) { | |
| $chunkEnd = $currentDate.AddHours(1) | |
| if ($chunkEnd -gt $ToDate) { | |
| $chunkEnd = $ToDate | |
| } | |
| $dateChunks.Add(@{ | |
| FromDate = $currentDate | |
| ToDate = $chunkEnd | |
| Index = $chunkIndex | |
| }) | |
| $chunkIndex++ | |
| $currentDate = $chunkEnd | |
| } | |
| Write-Host "Split $([math]::Round($totalHours, 1)) hours into $($dateChunks.Count) chunks (1 hour each)" -ForegroundColor Cyan | |
| # Store session cookies as a serializable format for parallel execution | |
| $cookieContainer = $script:session.Cookies | |
| $cookies = $cookieContainer.GetCookies([Uri]"https://security.microsoft.com") | |
| $cookieData = @() | |
| foreach ($cookie in $cookies) { | |
| $cookieData += @{ | |
| Name = $cookie.Name | |
| Value = $cookie.Value | |
| Domain = $cookie.Domain | |
| Path = $cookie.Path | |
| } | |
| } | |
| $headersData = @{} | |
| foreach ($key in $script:headers.Keys) { | |
| $headersData[$key] = $script:headers[$key] | |
| } | |
| try { | |
| Write-Verbose "Starting parallel retrieval of $($dateChunks.Count) chunk(s) with throttle limit of $ThrottleLimit" | |
| # Process chunks in parallel using ForEach-Object -Parallel (PowerShell 7+) | |
| if ($PSVersionTable.PSVersion.Major -ge 7) { | |
| $results = $dateChunks | ForEach-Object -ThrottleLimit $ThrottleLimit -Parallel { | |
| $chunk = $_ | |
| $deviceId = $using:deviceIdentifier | |
| $baseParams = $using:baseQueryParams | |
| $tempPath = $using:runTempPath | |
| $cookieInfo = $using:cookieData | |
| $headerInfo = $using:headersData | |
| $chunkFromDate = $chunk.FromDate | |
| $chunkToDate = $chunk.ToDate | |
| $chunkIndex = $chunk.Index | |
| # Recreate web session with cookies | |
| $webSession = [Microsoft.PowerShell.Commands.WebRequestSession]::new() | |
| foreach ($c in $cookieInfo) { | |
| $cookie = [System.Net.Cookie]::new($c.Name, $c.Value, $c.Path, $c.Domain) | |
| $webSession.Cookies.Add($cookie) | |
| } | |
| # Build query parameters for this chunk | |
| $correlationId = [guid]::NewGuid().ToString() | |
| $queryParams = @( | |
| "generateIdentityEvents=$($baseParams.GenerateIdentityEvents.ToString().ToLower())" | |
| "includeIdentityEvents=$($baseParams.IncludeIdentityEvents.ToString().ToLower())" | |
| "supportMdiOnlyEvents=$($baseParams.SupportMdiOnlyEvents.ToString().ToLower())" | |
| "fromDate=$([System.Uri]::EscapeDataString($chunkFromDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ')))" | |
| "toDate=$([System.Uri]::EscapeDataString($chunkToDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ')))" | |
| "correlationId=$correlationId" | |
| "doNotUseCache=$($baseParams.DoNotUseCache.ToString().ToLower())" | |
| "forceUseCache=$($baseParams.ForceUseCache.ToString().ToLower())" | |
| "pageSize=$($baseParams.PageSize)" | |
| "includeSentinelEvents=$($baseParams.IncludeSentinelEvents.ToString().ToLower())" | |
| ) | |
| if ($baseParams.MachineDnsName) { | |
| $queryParams = @("machineDnsName=$([System.Uri]::EscapeDataString($baseParams.MachineDnsName))") + $queryParams | |
| } | |
| if ($baseParams.SenseClientVersion) { | |
| $queryParams = @("SenseClientVersion=$([System.Uri]::EscapeDataString($baseParams.SenseClientVersion))") + $queryParams | |
| } | |
| if ($baseParams.MarkedEventsOnly) { | |
| $queryParams = @("markedEventsOnly=true") + $queryParams | |
| } | |
| $chunkEvents = [System.Collections.Generic.List[object]]::new() | |
| $Uri = "https://security.microsoft.com/apiproxy/mtp/mdeTimelineExperience/machines/$deviceId/events/?$($queryParams -join '&')" | |
| $maxRetries = 10 | |
| $baseDelay = 30 | |
| try { | |
| # Start timing this chunk | |
| $chunkStopwatch = [System.Diagnostics.Stopwatch]::StartNew() | |
| $pagesRetrieved = 0 | |
| do { | |
| $attempt = 0 | |
| $success = $false | |
| while (-not $success -and $attempt -lt $maxRetries) { | |
| try { | |
| $attempt++ | |
| $response = Invoke-RestMethod -Uri $Uri -ContentType "application/json" -WebSession $webSession -Headers $headerInfo -ErrorAction Stop | |
| $success = $true | |
| $pagesRetrieved++ | |
| } catch { | |
| $statusCode = $null | |
| if ($_.Exception.Response) { | |
| $statusCode = [int]$_.Exception.Response.StatusCode | |
| } | |
| if ($statusCode -eq 429 -or $statusCode -eq 403) { | |
| # Rate limited - use exponential backoff | |
| $delay = $baseDelay * [Math]::Pow(2, $attempt - 1) + (Get-Random -Minimum 1 -Maximum 10) | |
| $delay = [Math]::Min($delay, 300) # Cap at 5 minutes | |
| Write-Warning "Chunk $chunkIndex : Rate limited (HTTP $statusCode). Waiting $([int]$delay) seconds before retry $attempt/$maxRetries..." | |
| Start-Sleep -Seconds $delay | |
| } elseif ($attempt -lt $maxRetries) { | |
| $delay = Get-Random -Minimum 5 -Maximum 15 | |
| Write-Warning "Chunk $chunkIndex : Attempt $attempt failed. Retrying in $delay seconds... Error: $_" | |
| Start-Sleep -Seconds $delay | |
| } else { | |
| throw "Chunk $chunkIndex : Failed after $maxRetries attempts. Last error: $_" | |
| } | |
| } | |
| } | |
| if ($response -and $response.Items) { | |
| $chunkEvents.AddRange($response.Items) | |
| } | |
| if ([string]::IsNullOrWhiteSpace($response.Prev)) { | |
| break | |
| } else { | |
| $Uri = "https://security.microsoft.com/apiproxy/mtp/mdeTimelineExperience$($response.Prev)" | |
| # Small delay between pagination requests | |
| Start-Sleep -Milliseconds (Get-Random -Minimum 500 -Maximum 1500) | |
| } | |
| } while ($true) | |
| # Stop timing | |
| $chunkStopwatch.Stop() | |
| $elapsedSeconds = $chunkStopwatch.Elapsed.TotalSeconds | |
| # Write results to JSON file | |
| $fileName = "chunk_{0:D4}_{1:yyyyMMdd_HHmmss}_{2:yyyyMMdd_HHmmss}.json" -f $chunkIndex, $chunkFromDate, $chunkToDate | |
| $filePath = Join-Path $tempPath $fileName | |
| $jsonContent = @{ | |
| ChunkIndex = $chunkIndex | |
| FromDate = $chunkFromDate.ToString('o') | |
| ToDate = $chunkToDate.ToString('o') | |
| EventCount = $chunkEvents.Count | |
| Events = $chunkEvents | |
| } | ConvertTo-Json -Depth 100 -Compress | |
| $jsonContent | Out-File -FilePath $filePath -Encoding utf8 | |
| $fileSizeKB = [math]::Round((Get-Item $filePath).Length / 1KB, 2) | |
| @{ | |
| ChunkIndex = $chunkIndex | |
| FilePath = $filePath | |
| EventCount = $chunkEvents.Count | |
| FromDate = $chunkFromDate | |
| ToDate = $chunkToDate | |
| Success = $true | |
| ElapsedSeconds = [math]::Round($elapsedSeconds, 2) | |
| PagesRetrieved = $pagesRetrieved | |
| FileSizeKB = $fileSizeKB | |
| } | |
| } catch { | |
| if ($chunkStopwatch) { $chunkStopwatch.Stop() } | |
| @{ | |
| ChunkIndex = $chunkIndex | |
| Success = $false | |
| Error = $_.ToString() | |
| FromDate = $chunkFromDate | |
| ToDate = $chunkToDate | |
| ElapsedSeconds = if ($chunkStopwatch) { [math]::Round($chunkStopwatch.Elapsed.TotalSeconds, 2) } else { 0 } | |
| } | |
| } | |
| } | |
| } else { | |
| # Fallback for PowerShell 5.1 using runspace pool | |
| # Define script block for runspace execution | |
| $ps5ScriptBlock = { | |
| param($chunk, $deviceId, $baseParams, $tempPath, $cookieInfo, $headerInfo) | |
| $chunkFromDate = $chunk.FromDate | |
| $chunkToDate = $chunk.ToDate | |
| $chunkIndex = $chunk.Index | |
| # Recreate web session with cookies | |
| $webSession = [Microsoft.PowerShell.Commands.WebRequestSession]::new() | |
| foreach ($c in $cookieInfo) { | |
| $cookie = [System.Net.Cookie]::new($c.Name, $c.Value, $c.Path, $c.Domain) | |
| $webSession.Cookies.Add($cookie) | |
| } | |
| # Build query parameters for this chunk | |
| $correlationId = [guid]::NewGuid().ToString() | |
| $queryParams = @( | |
| "generateIdentityEvents=$($baseParams.GenerateIdentityEvents.ToString().ToLower())" | |
| "includeIdentityEvents=$($baseParams.IncludeIdentityEvents.ToString().ToLower())" | |
| "supportMdiOnlyEvents=$($baseParams.SupportMdiOnlyEvents.ToString().ToLower())" | |
| "fromDate=$([System.Uri]::EscapeDataString($chunkFromDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ')))" | |
| "toDate=$([System.Uri]::EscapeDataString($chunkToDate.ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ss.fffZ')))" | |
| "correlationId=$correlationId" | |
| "doNotUseCache=$($baseParams.DoNotUseCache.ToString().ToLower())" | |
| "forceUseCache=$($baseParams.ForceUseCache.ToString().ToLower())" | |
| "pageSize=$($baseParams.PageSize)" | |
| "includeSentinelEvents=$($baseParams.IncludeSentinelEvents.ToString().ToLower())" | |
| ) | |
| if ($baseParams.MachineDnsName) { | |
| $queryParams = @("machineDnsName=$([System.Uri]::EscapeDataString($baseParams.MachineDnsName))") + $queryParams | |
| } | |
| if ($baseParams.SenseClientVersion) { | |
| $queryParams = @("SenseClientVersion=$([System.Uri]::EscapeDataString($baseParams.SenseClientVersion))") + $queryParams | |
| } | |
| if ($baseParams.MarkedEventsOnly) { | |
| $queryParams = @("markedEventsOnly=true") + $queryParams | |
| } | |
| $chunkEvents = [System.Collections.Generic.List[object]]::new() | |
| $Uri = "https://security.microsoft.com/apiproxy/mtp/mdeTimelineExperience/machines/$deviceId/events/?$($queryParams -join '&')" | |
| $maxRetries = 10 | |
| $baseDelay = 30 | |
| try { | |
| # Start timing this chunk | |
| $chunkStopwatch = [System.Diagnostics.Stopwatch]::StartNew() | |
| $pagesRetrieved = 0 | |
| do { | |
| $attempt = 0 | |
| $success = $false | |
| while (-not $success -and $attempt -lt $maxRetries) { | |
| try { | |
| $attempt++ | |
| $response = Invoke-RestMethod -Uri $Uri -ContentType "application/json" -WebSession $webSession -Headers $headerInfo -ErrorAction Stop | |
| $success = $true | |
| $pagesRetrieved++ | |
| } catch { | |
| $statusCode = $null | |
| if ($_.Exception.Response) { | |
| $statusCode = [int]$_.Exception.Response.StatusCode | |
| } | |
| if ($statusCode -eq 429 -or $statusCode -eq 403) { | |
| $delay = $baseDelay * [Math]::Pow(2, $attempt - 1) + (Get-Random -Minimum 1 -Maximum 10) | |
| $delay = [Math]::Min($delay, 300) | |
| Start-Sleep -Seconds $delay | |
| } elseif ($attempt -lt $maxRetries) { | |
| $delay = Get-Random -Minimum 5 -Maximum 15 | |
| Start-Sleep -Seconds $delay | |
| } else { | |
| throw "Chunk $chunkIndex : Failed after $maxRetries attempts. Last error: $_" | |
| } | |
| } | |
| } | |
| if ($response -and $response.Items) { | |
| $chunkEvents.AddRange($response.Items) | |
| } | |
| if ([string]::IsNullOrWhiteSpace($response.Prev)) { | |
| break | |
| } else { | |
| $Uri = "https://security.microsoft.com/apiproxy/mtp/mdeTimelineExperience$($response.Prev)" | |
| Start-Sleep -Milliseconds (Get-Random -Minimum 500 -Maximum 1500) | |
| } | |
| } while ($true) | |
| # Stop timing | |
| $chunkStopwatch.Stop() | |
| $elapsedSeconds = $chunkStopwatch.Elapsed.TotalSeconds | |
| # Write results to JSON file | |
| $fileName = "chunk_{0:D4}_{1:yyyyMMdd_HHmmss}_{2:yyyyMMdd_HHmmss}.json" -f $chunkIndex, $chunkFromDate, $chunkToDate | |
| $filePath = Join-Path $tempPath $fileName | |
| $jsonContent = @{ | |
| ChunkIndex = $chunkIndex | |
| FromDate = $chunkFromDate.ToString('o') | |
| ToDate = $chunkToDate.ToString('o') | |
| EventCount = $chunkEvents.Count | |
| Events = $chunkEvents | |
| } | ConvertTo-Json -Depth 100 -Compress | |
| $jsonContent | Out-File -FilePath $filePath -Encoding utf8 | |
| $fileSizeKB = [math]::Round((Get-Item $filePath).Length / 1KB, 2) | |
| @{ | |
| ChunkIndex = $chunkIndex | |
| FilePath = $filePath | |
| EventCount = $chunkEvents.Count | |
| FromDate = $chunkFromDate | |
| ToDate = $chunkToDate | |
| Success = $true | |
| ElapsedSeconds = [math]::Round($elapsedSeconds, 2) | |
| PagesRetrieved = $pagesRetrieved | |
| FileSizeKB = $fileSizeKB | |
| } | |
| } catch { | |
| if ($chunkStopwatch) { $chunkStopwatch.Stop() } | |
| @{ | |
| ChunkIndex = $chunkIndex | |
| Success = $false | |
| Error = $_.ToString() | |
| FromDate = $chunkFromDate | |
| ToDate = $chunkToDate | |
| ElapsedSeconds = if ($chunkStopwatch) { [math]::Round($chunkStopwatch.Elapsed.TotalSeconds, 2) } else { 0 } | |
| } | |
| } | |
| } | |
| $runspacePool = [runspacefactory]::CreateRunspacePool(1, $ThrottleLimit) | |
| $runspacePool.Open() | |
| $jobs = @() | |
| foreach ($chunk in $dateChunks) { | |
| $powershell = [powershell]::Create() | |
| $powershell.RunspacePool = $runspacePool | |
| [void]$powershell.AddScript($ps5ScriptBlock) | |
| [void]$powershell.AddParameter('chunk', $chunk) | |
| [void]$powershell.AddParameter('deviceId', $deviceIdentifier) | |
| [void]$powershell.AddParameter('baseParams', $baseQueryParams) | |
| [void]$powershell.AddParameter('tempPath', $runTempPath) | |
| [void]$powershell.AddParameter('cookieInfo', $cookieData) | |
| [void]$powershell.AddParameter('headerInfo', $headersData) | |
| $jobs += @{ | |
| PowerShell = $powershell | |
| Handle = $powershell.BeginInvoke() | |
| Chunk = $chunk | |
| } | |
| } | |
| # Wait for all jobs and collect results | |
| $results = @() | |
| foreach ($job in $jobs) { | |
| try { | |
| $result = $job.PowerShell.EndInvoke($job.Handle) | |
| $results += $result | |
| } catch { | |
| Write-Warning "Chunk $($job.Chunk.Index) failed: $_" | |
| $results += @{ | |
| ChunkIndex = $job.Chunk.Index | |
| Success = $false | |
| Error = $_.ToString() | |
| } | |
| } finally { | |
| $job.PowerShell.Dispose() | |
| } | |
| } | |
| $runspacePool.Close() | |
| $runspacePool.Dispose() | |
| } | |
| # Check for failures | |
| $failures = $results | Where-Object { -not $_.Success } | |
| if ($failures) { | |
| Write-Warning "Some chunks failed to retrieve: $($failures.ChunkIndex -join ', ')" | |
| } | |
| # Output timing information for each chunk (standard output) | |
| Write-Host "`n=== Chunk Download Statistics ===" -ForegroundColor Yellow | |
| $totalElapsed = 0 | |
| $totalEvents = 0 | |
| $totalSizeKB = 0 | |
| $maxElapsed = 0 | |
| foreach ($result in ($results | Sort-Object ChunkIndex)) { | |
| $dateRange = "{0:yyyy-MM-dd HH:mm} to {1:yyyy-MM-dd HH:mm}" -f $result.FromDate, $result.ToDate | |
| if ($result.Success) { | |
| $totalElapsed += $result.ElapsedSeconds | |
| $totalEvents += $result.EventCount | |
| $totalSizeKB += $result.FileSizeKB | |
| if ($result.ElapsedSeconds -gt $maxElapsed) { $maxElapsed = $result.ElapsedSeconds } | |
| $eventsPerSec = if ($result.ElapsedSeconds -gt 0) { [math]::Round($result.EventCount / $result.ElapsedSeconds, 1) } else { 0 } | |
| Write-Host "Chunk $($result.ChunkIndex): $dateRange | Events: $($result.EventCount) | Pages: $($result.PagesRetrieved) | Size: $($result.FileSizeKB) KB | Time: $($result.ElapsedSeconds)s | Rate: $eventsPerSec events/sec" | |
| } else { | |
| Write-Host "Chunk $($result.ChunkIndex): $dateRange | FAILED after $($result.ElapsedSeconds)s - $($result.Error)" -ForegroundColor Red | |
| } | |
| } | |
| $overallEventsPerSec = if ($maxElapsed -gt 0) { [math]::Round($totalEvents / $maxElapsed, 1) } else { 0 } | |
| Write-Host "=== Summary ===" -ForegroundColor Yellow | |
| Write-Host "Total chunks: $($results.Count) | Total events: $totalEvents | Total size: $([math]::Round($totalSizeKB / 1024, 2)) MB" | |
| Write-Host "Cumulative download time: $([math]::Round($totalElapsed, 2))s | Wall-clock time: $([math]::Round($maxElapsed, 2))s | Effective rate: $overallEventsPerSec events/sec" -ForegroundColor Green | |
| # Merge all JSON files | |
| Write-Verbose "Merging results from $($results.Count) chunk(s)..." | |
| $allEvents = [System.Collections.Generic.List[object]]::new() | |
| $jsonFiles = Get-ChildItem -Path $runTempPath -Filter "chunk_*.json" | Sort-Object Name | |
| foreach ($file in $jsonFiles) { | |
| $chunkData = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json | |
| if ($chunkData.Events) { | |
| foreach ($event in $chunkData.Events) { | |
| $allEvents.Add($event) | |
| } | |
| } | |
| } | |
| Write-Verbose "Total events retrieved: $($allEvents.Count)" | |
| # Clean up temp files unless KeepTempFiles is specified | |
| if (-not $KeepTempFiles) { | |
| Write-Verbose "Cleaning up temporary files..." | |
| Remove-Item -Path $runTempPath -Recurse -Force -ErrorAction SilentlyContinue | |
| } else { | |
| Write-Verbose "Temporary files kept at: $runTempPath" | |
| } | |
| # Return events sorted by timestamp (if available) | |
| if ($allEvents.Count -gt 0 -and $allEvents[0].PSObject.Properties['Timestamp']) { | |
| return $allEvents | Sort-Object -Property Timestamp -Descending | |
| } | |
| return $allEvents | |
| } catch { | |
| Write-Error "Failed to retrieve endpoint device timeline: $_" | |
| } | |
| } | |
| end { | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment