diegoeche

Battle Report: SQL Server Meltdown & Recovery — March 9, 2026

The Scene

Sunday morning. Merchants can't load review widgets. Dashboard inaccessible. Churn reported. The legacy .NET monolith (app-backend-api) running on a self-managed SQL Server (r6a.16xlarge — 64 vCPUs, 512 GB RAM) was completely unresponsive.

What followed was a 12-hour debugging marathon that uncovered five cascading failures, a year-old regression, and a SQL Server death spiral that required a force-kill to resolve.

---

diegoeche

k8s-mvp Environment Structure

How the k8s-mvp repository environments are structured using Kustomize overlays and GitOps.

Directory Layout

clusters/platform/
├── base/                          # Shared service definitions
│   ├── app-backend-api/

diegoeche

Models::Audience.first
=> #<FormsHub::Models::Audience @values={:id=>"ccdd57f2-d24f-4ff7-9356-b41f82756c94", :iid=>17, :account_id=>"0166cbcb-7827-4698-a267-fec98676db0b", :type=>"company", :name=>"Paid companies", :description=>nil, :facebook_account_id=>"act_10150428762524283", :facebook_audience_id=>"6103550209001", :company_segment_id=>"c6eddfbe-23ff-4287-b31c-9d3291c89da3", :person_segment_id=>nil, :company_exclusion_segment_id=>nil, :person_exclusion_segment_id=>nil, :role=>nil, :seniority=>nil, :search=>nil, :path=>nil, :status=>"active", :created_at=>2018-04-04 02:05:48 UTC, :updated_at=>2018-10-11 10:42:55 UTC, :last_sync_at=>nil, :archived_at=>nil}>

diegoeche

const axios = require('axios')
const sleep = require('sleep')
const base64 = require('base-64')

const appId = process.env["GEENY_APPLICATION_ID"]
const host = process.env["GEENY_APPLICATION_BROKER_URL"]

const brokerConfig = {
  appId: appId,
  messageTypeId: '54121087-14f1-4c2a-835f-117681618cc9', // incoming Develco messageType

diegoeche

const axios = require('axios')
const sleep = require('sleep')
const base64 = require('base-64')

const appId = process.env["GEENY_APPLICATION_ID"]
const host = process.env["GEENY_APPLICATION_BROKER_URL"]

const brokerConfig = {
  appId: appId,
  messageTypeId: '54121087-14f1-4c2a-835f-117681618cc9', // incoming Develco messageType

diegoeche

static bool STATS = false;
static const int FRAME_PLOT_SIZE = 50;
static float frameTimes[FRAME_PLOT_SIZE];

void renderUIElements(float elapsed) {
  if (STATS) {
    std::ostringstream os;
    os << "T: " << elapsed;
    std::string str = os.str();
    char * cstr = new char [str.length()+1];

diegoeche

require "digest/md5"
require "openssl"
require "securerandom"
require 'benchmark'
require "base64"

SIZE = 1000
def create_test
  ret = []
  SIZE.times do

diegoeche

-- Before caching RoF fitness
-- 00:00:00.1983681

RoF Fitness: 	00:00:00.0025516
Fitness total: 	00:00:00.0055062
Actions: 	00:00:00.0664216
Playing total: 	00:00:00.0255507

-- After caching RoF fitness
-- 00:00:00.2025582

diegoeche

public static TResult Aggregate<TSource, TAccumulate, TResult> (this IEnumerable<TSource> source, TAccumulate seed, Func<TAccumulate, TSource, TAccumulate> func, Func<TAccumulate, TResult> resultSelector) {
       return System.Linq.Aggregate(source, TAccumulate seed, Func<TAccumulate, TSource, TAccumulate> func, Func<TAccumulate, TResult> resultSelector) ;
}

public static TAccumulate Aggregate<TSource, TAccumulate> (this IEnumerable<TSource> source, TAccumulate seed, Func<TAccumulate, TSource, TAccumulate> func) {
       return System.Linq.Aggregate (source, TAccumulate seed, Func<TAccumulate, TSource, TAccumulate> func);
}

public static bool All<TSource> (this IEnumerable<TSource> source, Func<TSource, bool> predicate) {

diegoeche

import Data.List

removeNegatives = filter $ not . (< 0) . head

removeEmpties = filter $ not . null

safeTail (x:xs) = xs
safeTail x = x

parts [] = []