Last active
August 11, 2025 09:43
-
-
Save alirezanet/b92d8e65b2b73724c8eea8ba890ba2a3 to your computer and use it in GitHub Desktop.
AWS SQS DLQ Downloader using .NET 10 single-file app. Downloads all messages from an SQS queue or DLQ to a .jsonl file, flushing to disk before deletion to avoid data loss. Supports AWS profile/region config, queue name or URL, and a --dry-run mode for safe testing without deleting messages.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // Run with: dotnet run sqs_dump.cs -- --queue-url https://sqs.eu-west-1.amazonaws.com/123456789012/my-dlq | |
| // Optional: --out dlq.jsonl --region eu-west-1 --profile default --visibility 300 --wait 20 --queue-name my-dlq --dry-run | |
| #:package AWSSDK.SQS@3.7.500.5 | |
| using Amazon; | |
| using Amazon.SQS; | |
| using Amazon.SQS.Model; | |
| using Amazon.Runtime; | |
| using Amazon.Runtime.CredentialManagement; | |
| using System.Text; | |
| using System.Text.Json; | |
| string? Arg(string name) | |
| { | |
| for (int i = 0; i < args.Length - 1; i++) | |
| if (args[i].Equals(name, StringComparison.OrdinalIgnoreCase)) | |
| return args[i + 1]; | |
| return null; | |
| } | |
| bool HasFlag(string flag) => | |
| args.Any(a => a.Equals(flag, StringComparison.OrdinalIgnoreCase)); | |
| var queueUrl = Arg("--queue-url") ?? Environment.GetEnvironmentVariable("QUEUE_URL"); | |
| var queueName = Arg("--queue-name"); | |
| var region = Arg("--region") ?? Environment.GetEnvironmentVariable("AWS_REGION") ?? "eu-west-1"; | |
| var profile = Arg("--profile") ?? Environment.GetEnvironmentVariable("AWS_PROFILE") ?? "default"; | |
| var outPath = Arg("--out") ?? $"sqs-dlq-{DateTimeOffset.UtcNow:yyyyMMdd-HHmmss}.jsonl"; | |
| var visibility = int.TryParse(Arg("--visibility") ?? "180", out var vis) ? vis : 180; | |
| var waitSecs = int.TryParse(Arg("--wait") ?? "20", out var wt) ? wt : 20; | |
| var dryRun = HasFlag("--dry-run"); | |
| var config = new AmazonSQSConfig(); | |
| if (!string.IsNullOrWhiteSpace(region)) | |
| config.RegionEndpoint = RegionEndpoint.GetBySystemName(region); | |
| AWSCredentials? creds = null; | |
| if (!string.IsNullOrWhiteSpace(profile) && | |
| new CredentialProfileStoreChain().TryGetAWSCredentials(profile, out var c)) | |
| creds = c; | |
| using var sqs = creds is null ? new AmazonSQSClient(config) : new AmazonSQSClient(creds, config); | |
| if (string.IsNullOrWhiteSpace(queueUrl) && !string.IsNullOrWhiteSpace(queueName)) | |
| { | |
| var urlResp = await sqs.GetQueueUrlAsync(new GetQueueUrlRequest { QueueName = queueName }); | |
| queueUrl = urlResp.QueueUrl; | |
| } | |
| if (string.IsNullOrWhiteSpace(queueUrl)) | |
| { | |
| Console.Error.WriteLine("Usage: dotnet run app.cs -- --queue-url <url> [--queue-name name] [--out file.jsonl] [--region eu-west-1] [--profile default] [--visibility 180] [--wait 20] [--dry-run]"); | |
| Environment.ExitCode = 2; | |
| return; | |
| } | |
| Console.WriteLine($"Downloading from: {queueUrl}"); | |
| Console.WriteLine($"Writing to : {outPath}"); | |
| if (dryRun) Console.WriteLine("Dry-run mode: No messages will be deleted."); | |
| var sysAttrs = new List<string> { "All" }; | |
| var msgAttrs = new List<string> { "All" }; | |
| using var fs = new FileStream(outPath, FileMode.Append, FileAccess.Write, FileShare.Read); | |
| using var writer = new StreamWriter(fs, new UTF8Encoding(false)) { AutoFlush = true }; | |
| var cancelled = false; | |
| Console.CancelKeyPress += (_, e) => { e.Cancel = true; cancelled = true; }; | |
| int total = 0, emptyPolls = 0, emptyLimit = 3; | |
| while (!cancelled) | |
| { | |
| var req = new ReceiveMessageRequest | |
| { | |
| QueueUrl = queueUrl!, | |
| MaxNumberOfMessages = 10, | |
| WaitTimeSeconds = waitSecs, | |
| VisibilityTimeout = visibility, | |
| MessageSystemAttributeNames = sysAttrs, | |
| MessageAttributeNames = msgAttrs | |
| }; | |
| var resp = await sqs.ReceiveMessageAsync(req); | |
| if (resp.Messages.Count == 0) | |
| { | |
| if (++emptyPolls >= emptyLimit) break; | |
| continue; | |
| } | |
| emptyPolls = 0; | |
| foreach (var m in resp.Messages) | |
| { | |
| var line = new | |
| { | |
| m.MessageId, | |
| m.MD5OfBody, | |
| m.MD5OfMessageAttributes, | |
| Body = m.Body, | |
| Attributes = m.Attributes, | |
| MessageAttributes = m.MessageAttributes?.ToDictionary( | |
| kv => kv.Key, | |
| kv => new { | |
| kv.Value.DataType, | |
| kv.Value.StringValue, | |
| BinaryValueBase64 = kv.Value.BinaryValue != null | |
| ? Convert.ToBase64String(kv.Value.BinaryValue.ToArray()) | |
| : null | |
| }) | |
| }; | |
| await writer.WriteLineAsync(JsonSerializer.Serialize(line)); | |
| writer.Flush(); | |
| fs.Flush(true); | |
| if (dryRun) | |
| { | |
| Console.WriteLine($"[Dry-run] Would delete: {m.MessageId}"); | |
| total++; | |
| } | |
| else | |
| { | |
| try | |
| { | |
| Console.WriteLine($"Deleting: {m.MessageId}"); | |
| await sqs.DeleteMessageAsync(queueUrl!, m.ReceiptHandle); | |
| total++; | |
| } | |
| catch (Exception ex) | |
| { | |
| Console.Error.WriteLine($"Delete failed for {m.MessageId}: {ex.Message}"); | |
| try { await sqs.ChangeMessageVisibilityAsync(queueUrl!, m.ReceiptHandle, Math.Min(visibility, 300)); } catch { } | |
| } | |
| } | |
| } | |
| } | |
| Console.WriteLine($"Done. {(dryRun ? "Tested" : "Downloaded and deleted")} {total} messages."); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment