Skip to content

Instantly share code, notes, and snippets.

@alirezanet
Last active August 11, 2025 09:43
Show Gist options
  • Select an option

  • Save alirezanet/b92d8e65b2b73724c8eea8ba890ba2a3 to your computer and use it in GitHub Desktop.

Select an option

Save alirezanet/b92d8e65b2b73724c8eea8ba890ba2a3 to your computer and use it in GitHub Desktop.
AWS SQS DLQ Downloader using .NET 10 single-file app. Downloads all messages from an SQS queue or DLQ to a .jsonl file, flushing to disk before deletion to avoid data loss. Supports AWS profile/region config, queue name or URL, and a --dry-run mode for safe testing without deleting messages.
// Run with: dotnet run sqs_dump.cs -- --queue-url https://sqs.eu-west-1.amazonaws.com/123456789012/my-dlq
// Optional: --out dlq.jsonl --region eu-west-1 --profile default --visibility 300 --wait 20 --queue-name my-dlq --dry-run
#:package AWSSDK.SQS@3.7.500.5
using Amazon;
using Amazon.SQS;
using Amazon.SQS.Model;
using Amazon.Runtime;
using Amazon.Runtime.CredentialManagement;
using System.Text;
using System.Text.Json;
string? Arg(string name)
{
for (int i = 0; i < args.Length - 1; i++)
if (args[i].Equals(name, StringComparison.OrdinalIgnoreCase))
return args[i + 1];
return null;
}
bool HasFlag(string flag) =>
args.Any(a => a.Equals(flag, StringComparison.OrdinalIgnoreCase));
var queueUrl = Arg("--queue-url") ?? Environment.GetEnvironmentVariable("QUEUE_URL");
var queueName = Arg("--queue-name");
var region = Arg("--region") ?? Environment.GetEnvironmentVariable("AWS_REGION") ?? "eu-west-1";
var profile = Arg("--profile") ?? Environment.GetEnvironmentVariable("AWS_PROFILE") ?? "default";
var outPath = Arg("--out") ?? $"sqs-dlq-{DateTimeOffset.UtcNow:yyyyMMdd-HHmmss}.jsonl";
var visibility = int.TryParse(Arg("--visibility") ?? "180", out var vis) ? vis : 180;
var waitSecs = int.TryParse(Arg("--wait") ?? "20", out var wt) ? wt : 20;
var dryRun = HasFlag("--dry-run");
var config = new AmazonSQSConfig();
if (!string.IsNullOrWhiteSpace(region))
config.RegionEndpoint = RegionEndpoint.GetBySystemName(region);
AWSCredentials? creds = null;
if (!string.IsNullOrWhiteSpace(profile) &&
new CredentialProfileStoreChain().TryGetAWSCredentials(profile, out var c))
creds = c;
using var sqs = creds is null ? new AmazonSQSClient(config) : new AmazonSQSClient(creds, config);
if (string.IsNullOrWhiteSpace(queueUrl) && !string.IsNullOrWhiteSpace(queueName))
{
var urlResp = await sqs.GetQueueUrlAsync(new GetQueueUrlRequest { QueueName = queueName });
queueUrl = urlResp.QueueUrl;
}
if (string.IsNullOrWhiteSpace(queueUrl))
{
Console.Error.WriteLine("Usage: dotnet run app.cs -- --queue-url <url> [--queue-name name] [--out file.jsonl] [--region eu-west-1] [--profile default] [--visibility 180] [--wait 20] [--dry-run]");
Environment.ExitCode = 2;
return;
}
Console.WriteLine($"Downloading from: {queueUrl}");
Console.WriteLine($"Writing to : {outPath}");
if (dryRun) Console.WriteLine("Dry-run mode: No messages will be deleted.");
var sysAttrs = new List<string> { "All" };
var msgAttrs = new List<string> { "All" };
using var fs = new FileStream(outPath, FileMode.Append, FileAccess.Write, FileShare.Read);
using var writer = new StreamWriter(fs, new UTF8Encoding(false)) { AutoFlush = true };
var cancelled = false;
Console.CancelKeyPress += (_, e) => { e.Cancel = true; cancelled = true; };
int total = 0, emptyPolls = 0, emptyLimit = 3;
while (!cancelled)
{
var req = new ReceiveMessageRequest
{
QueueUrl = queueUrl!,
MaxNumberOfMessages = 10,
WaitTimeSeconds = waitSecs,
VisibilityTimeout = visibility,
MessageSystemAttributeNames = sysAttrs,
MessageAttributeNames = msgAttrs
};
var resp = await sqs.ReceiveMessageAsync(req);
if (resp.Messages.Count == 0)
{
if (++emptyPolls >= emptyLimit) break;
continue;
}
emptyPolls = 0;
foreach (var m in resp.Messages)
{
var line = new
{
m.MessageId,
m.MD5OfBody,
m.MD5OfMessageAttributes,
Body = m.Body,
Attributes = m.Attributes,
MessageAttributes = m.MessageAttributes?.ToDictionary(
kv => kv.Key,
kv => new {
kv.Value.DataType,
kv.Value.StringValue,
BinaryValueBase64 = kv.Value.BinaryValue != null
? Convert.ToBase64String(kv.Value.BinaryValue.ToArray())
: null
})
};
await writer.WriteLineAsync(JsonSerializer.Serialize(line));
writer.Flush();
fs.Flush(true);
if (dryRun)
{
Console.WriteLine($"[Dry-run] Would delete: {m.MessageId}");
total++;
}
else
{
try
{
Console.WriteLine($"Deleting: {m.MessageId}");
await sqs.DeleteMessageAsync(queueUrl!, m.ReceiptHandle);
total++;
}
catch (Exception ex)
{
Console.Error.WriteLine($"Delete failed for {m.MessageId}: {ex.Message}");
try { await sqs.ChangeMessageVisibilityAsync(queueUrl!, m.ReceiptHandle, Math.Min(visibility, 300)); } catch { }
}
}
}
}
Console.WriteLine($"Done. {(dryRun ? "Tested" : "Downloaded and deleted")} {total} messages.");
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment