Skip to content

Instantly share code, notes, and snippets.

@sibljon

sibljon/components.json

Created November 11, 2025 12:40
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save sibljon/184637dcf82463714b00d3be15c43a61 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save sibljon/184637dcf82463714b00d3be15c43a61 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
components.json
{
"page": {
"id": "qvq7h8gxchl0",
"name": "Spruce Health",
"url": "https://status.sprucehealth.com",
"time_zone": "America/Los_Angeles",
"updated_at": "2025-11-10T23:41:11.268-08:00"
},
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "5xhfkvp5p5jh",
"name": "Marketing Website",
"status": "operational",
"created_at": "2017-03-07T14:46:00.105-08:00",
"updated_at": "2019-02-13T15:14:40.743-08:00",
"position": 4,
"description": "https://sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "6wxpvnqpkbsg",
"name": "Fax",
"status": "operational",
"created_at": "2018-08-14T18:45:20.310-07:00",
"updated_at": "2025-10-28T11:58:43.083-07:00",
"position": 5,
"description": "Inbound/Outbound Digital Fax",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "nftn1q0cs42j",
"name": "Spruce app notifications",
"status": "operational",
"created_at": "2021-12-10T13:25:35.080-08:00",
"updated_at": "2025-10-20T14:35:37.106-07:00",
"position": 8,
"description": "Real time notifications for new messages and video calls.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xdrmpzwhxky9",
"name": "Integrations",
"status": "operational",
"created_at": "2022-10-12T08:39:52.774-07:00",
"updated_at": "2025-10-20T12:50:00.997-07:00",
"position": 9,
"description": "This component comprises all native integrations between Spruce and external systems, such as Spruce-Elation, Spruce-Hint and Spruce-Stripe integrations.",
"showcase": true,
"start_date": "2022-10-12",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "yvdsvc2s20x7",
"name": "API",
"status": "operational",
"created_at": "2023-01-05T12:37:47.464-08:00",
"updated_at": "2025-10-20T12:50:01.014-07:00",
"position": 10,
"description": "https://api.sprucehealth.com",
"showcase": true,
"start_date": "2023-01-05",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
]
}
Raw
incidents.json
{
"page": {
"id": "qvq7h8gxchl0",
"name": "Spruce Health",
"url": "https://status.sprucehealth.com",
"time_zone": "America/Los_Angeles",
"updated_at": "2025-11-10T23:41:11.268-08:00"
},
"incidents": [
{
"id": "fczp1zl5wbnc",
"name": "Spruce platform issues",
"status": "resolved",
"created_at": "2025-10-28T11:49:32.984-07:00",
"updated_at": "2025-10-29T21:18:47.615-07:00",
"monitoring_at": "2025-10-28T11:58:43.000-07:00",
"resolved_at": "2025-10-29T21:18:47.601-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/h8p5txpc1mm3",
"started_at": "2025-10-28T11:49:32.977-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "xpz65jz8smhh",
"status": "resolved",
"body": "The issue has been resolved, and the platform is fully functional. Impact occurred between 11:33 AM and 11:48 AM PT. We’ve identified the root cause and will be discussing ways to reduce the likelihood of similar issues in the future.",
"incident_id": "fczp1zl5wbnc",
"created_at": "2025-10-29T21:18:47.601-07:00",
"updated_at": "2025-10-29T21:18:47.601-07:00",
"display_at": "2025-10-29T21:18:47.601-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "x2vp7rtys1zp",
"status": "monitoring",
"body": "The issue has been resolved, and the platform is currently functional. Impact occurred between 11:33 AM and 11:48 AM PT. We’re continuing to investigate the root cause and will monitor closely in case the issue reoccurs.",
"incident_id": "fczp1zl5wbnc",
"created_at": "2025-10-28T11:58:43.127-07:00",
"updated_at": "2025-10-28T12:28:35.603-07:00",
"display_at": "2025-10-28T11:58:43.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "y9glw51p5rgc",
"status": "investigating",
"body": "We're currently investigating issues accessing the Spruce platform. We're investigating and will update this page as we learn more",
"incident_id": "fczp1zl5wbnc",
"created_at": "2025-10-28T11:49:33.144-07:00",
"updated_at": "2025-10-28T11:49:33.144-07:00",
"display_at": "2025-10-28T11:49:33.144-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "6wxpvnqpkbsg",
"name": "Fax",
"status": "operational",
"created_at": "2018-08-14T18:45:20.310-07:00",
"updated_at": "2025-10-28T11:58:43.083-07:00",
"position": 5,
"description": "Inbound/Outbound Digital Fax",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "p4p8drbj3q4d",
"name": "[RESOLVED] Cloud Provider (AWS) Outage",
"status": "resolved",
"created_at": "2025-10-20T00:41:24.000-07:00",
"updated_at": "2025-10-20T21:31:50.155-07:00",
"monitoring_at": "2025-10-20T04:55:10.349-07:00",
"resolved_at": "2025-10-20T21:31:50.133-07:00",
"impact": "major",
"shortlink": "https://stspg.io/c7z8dzf2zpgh",
"started_at": "2025-10-20T00:41:24.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "q0mdfrmkm1gm",
"status": "resolved",
"body": "AWS has confirmed full recovery of their services, and Twilio has also reported full restoration of functionality.\n\nAll Spruce systems are operating normally. The inbox, calls, secure messaging, video visits, faxes, and SMS are functioning as expected, and notifications are being delivered in real time.\n\nWith all systems stable and fully operational, we are considering this incident resolved.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T21:31:50.133-07:00",
"updated_at": "2025-10-20T21:31:50.133-07:00",
"display_at": "2025-10-20T21:31:50.133-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "3c7x32zhdz04",
"status": "monitoring",
"body": "We are seeing continued signs of recovery from AWS.\n\nOur push notification delivery system has caught up, and the Spruce Backup System has completed processing all pending notifications.\n\nSearch-related events have also caught up, and the system is now running at its intended capacity to fully process requests.\n\nWe are reviewing patient and provider accounts created during the outage period to ensure that all data is intact and consistent.\n\nOur communication infrastructure provider, Twilio, which also depends on AWS, continues to work toward full recovery. As they restore their services, inbound and outbound SMS may still experience delays, and call connections may occasionally be impacted.\n\nWe are closely monitoring the recovery of both AWS and Twilio to ensure that Spruce remains fully operational and stable.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T14:35:37.137-07:00",
"updated_at": "2025-10-20T14:35:37.137-07:00",
"display_at": "2025-10-20T14:35:37.137-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "qd6wnd8zypjc",
"status": "monitoring",
"body": "Our systems are no longer experiencing delays in processing call events, outbound SMS, faxes, or emails. We have caught up on the backlog, and any previously delayed items should now appear in the inbox with a “delayed message” indicator.\n\nAWS has reported early signs of recovery, and we are also observing an increase in our system capacity as a result of prior scaling efforts.\n\nAt this time, the Spruce inbox remains fully functional. Calls, secure messages, video visits, and related events are operating normally, and call events, emails, faxes, and SMS messages should now be delivered in a timely manner.\n\nThere are still delays in processing push notifications and a backlog of search-related events. As a result, some users may experience delayed or missing push notifications, and search results or contact lists may appear stale or out of date. We are monitoring both areas closely to ensure that they continue to process and catch up over time.\n\nThe backup system has been turned off now that there are no longer delays in processing inbound or outbound events.\n\nWe will continue to closely monitor the situation to ensure the platform remains stable and fully restored.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T12:50:01.053-07:00",
"updated_at": "2025-10-20T12:50:01.053-07:00",
"display_at": "2025-10-20T12:50:01.053-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "mj1nmnny8z13",
"status": "monitoring",
"body": "The Spruce inbox remains operational, and you can continue to make and receive calls, exchange secure messages, and conduct video visits.\n\nWe are, however, experiencing additional delays in processing certain system events due to new issues reported by our cloud provider, AWS. Another part of their infrastructure that Spruce depends on has been impacted.\n\n• Push notifications: There is a delay in processing push notifications, which may prevent patients and providers from receiving real-time alerts for new messages or updates.\n• Backup system events: There is a delay in processing Spruce Backup system events, resulting in delayed delivery of email and SMS notifications related to inbound SMS and calls.\n\nIn relative good news, we’ve been able to fully catch up and remain up to date with inbound SMS, email, and fax processing. Our team is now working to catch up on outbound SMS, call events, fax, and email in a similar manner.\n\nAWS continues to provide updates, but none have indicated signs of recovery yet. We are continuing to closely monitor the situation and will share updates as they become available.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T11:30:30.595-07:00",
"updated_at": "2025-10-20T11:30:30.595-07:00",
"display_at": "2025-10-20T11:30:30.595-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "nkszmvv6mtvj",
"status": "monitoring",
"body": "The Spruce inbox, as well as the ability to make and receive calls, send and receive secure messages, and conduct video visits, all remain currently functional.\n\nThere continue to be significant, multi-hour delays in processing inbound and outbound call events, SMS messages, faxes, emails, and search-related updates. These delays have persisted for some time, and we are unable to increase processing capacity given the ongoing issues with our cloud provider, AWS.\n\nWe are continuing to closely monitor the situation. While AWS has been providing updates, none have indicated signs of recovery. As a result, we cannot guarantee that services will remain stable until more progress is reported.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T10:37:03.704-07:00",
"updated_at": "2025-10-20T10:37:03.704-07:00",
"display_at": "2025-10-20T10:37:03.704-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "4pvm9qs3qghh",
"status": "monitoring",
"body": "As of 8:05 AM PT, the Spruce inbox is loading and operational. Inbound and outbound calls, as well as video visits, are currently functional.\n\nThere continue to be delays in processing inbound and outbound call events, SMS messages, faxes, and emails.\n\nPlease note that AWS, our cloud provider, has not yet provided an update on their recovery. As a result, platform stability cannot be guaranteed at this time. We are continuing to monitor the situation closely and will share updates as they become available.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T08:45:10.197-07:00",
"updated_at": "2025-10-20T08:45:10.197-07:00",
"display_at": "2025-10-20T08:45:10.197-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "h4c7pybt75v9",
"status": "monitoring",
"body": "Our cloud provider (AWS) is currently experiencing widespread issues that are impacting the availability and performance of the Spruce platform. We are closely monitoring the situation and working to restore full functionality.\n\nAt this time, you may experience disruptions across multiple areas of Spruce, including loading the inbox, sending and receiving messages, calls, SMS, faxes, and video visits. Some features may work intermittently or with delays.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T07:58:30.874-07:00",
"updated_at": "2025-10-20T07:58:30.874-07:00",
"display_at": "2025-10-20T07:58:30.874-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "v6hvpzcd1t07",
"status": "monitoring",
"body": "We have enabled the Spruce backup system (second time around during this incident) as of 7:21am PT. The backup system will help notify those that have subscribed to updates of any incoming calls and texts. You can learn more about the backup system here: https://help.sprucehealth.com/hc/en-us/articles/23003270604699-Spruce-Backup-System\n\nThe Spruce inbox is intermittently loading at the moment.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T07:28:42.914-07:00",
"updated_at": "2025-10-20T07:28:42.914-07:00",
"display_at": "2025-10-20T07:28:42.914-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "vg8hhzyr4230",
"status": "monitoring",
"body": "Our cloud provider AWS continues to experience operational issues which is delaying our recovery. Our system is currently working at partial capacity in servicing requests.\n\nOutbound fax and SMS are delayed. Inbound and outbound calls are functional without guarantees of full recovery yet.\n\nIn general, you will continue to experience intermittent issues with Spruce given the limited capacity in which the platform is being serviced.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T07:17:54.551-07:00",
"updated_at": "2025-10-20T07:17:54.551-07:00",
"display_at": "2025-10-20T07:17:54.551-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "b64rpqk6qdsn",
"status": "monitoring",
"body": "We're seeing recovery across all systems. Our telephony partner is still reporting that there could be elevated rates of dropped calls, and that SMS may be delayed, but the majority of calls and SMS are now working normally. We will continue to provide updated information here as it becomes available.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T06:14:38.325-07:00",
"updated_at": "2025-10-20T06:14:38.325-07:00",
"display_at": "2025-10-20T06:14:38.325-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "5hknqynrb3jq",
"status": "monitoring",
"body": "We're seeing strong signs of recovery. Phone and video calls are now functional, and most users can once again send and receive SMS messages.\n\nSome delays in receiving SMS messages may still occur as our telephony partner continues their recovery efforts.\n\nOur team is closely monitoring the situation and will share additional updates as more information becomes available.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T04:55:10.349-07:00",
"updated_at": "2025-10-20T04:55:10.349-07:00",
"display_at": "2025-10-20T04:55:10.349-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "major_outage",
"new_status": "operational"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "1rcl1m8fqz5h",
"status": "identified",
"body": "AWS has restored most service, and our telephony partner is beginning to restore service. The Spruce inbox is now accessible and secure messaging is working. Phone and SMS continue to be impacted. At this time we still expect that SMS sent and received during the outage will be delivered after our telephony partner's service has been restored.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T03:33:37.296-07:00",
"updated_at": "2025-10-20T03:33:37.296-07:00",
"display_at": "2025-10-20T03:33:37.296-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "major_outage",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "major_outage",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "major_outage",
"new_status": "degraded_performance"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "major_outage",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "ts5s5cqhcffj",
"status": "investigating",
"body": "We are continuing to investigate workarounds, however our telephony infrastructure partner is also affected by the major AWS outage and as such cannot deliver phone calls or SMS to our system, so we cannot provide the typical backup system coverage that we are normally able to provide during a Spruce outage. AWS has reported that they have identified the root cause of the issue and are working on a resolution.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T02:06:09.728-07:00",
"updated_at": "2025-10-20T02:06:09.728-07:00",
"display_at": "2025-10-20T02:06:09.728-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "major_outage",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "xmy0968rgnf4",
"status": "investigating",
"body": "Our cloud provider AWS is experiencing a major outage that is affecting the ability to log into Spruce, and for us to route phone calls. All messages received during this time will be delivered when the system recovers. We are currently investigating the full impact.",
"incident_id": "p4p8drbj3q4d",
"created_at": "2025-10-20T00:41:24.726-07:00",
"updated_at": "2025-10-20T00:41:39.118-07:00",
"display_at": "2025-10-20T00:41:24.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "major_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "major_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "major_outage"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "major_outage"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "6wxpvnqpkbsg",
"name": "Fax",
"status": "operational",
"created_at": "2018-08-14T18:45:20.310-07:00",
"updated_at": "2025-10-28T11:58:43.083-07:00",
"position": 5,
"description": "Inbound/Outbound Digital Fax",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "nftn1q0cs42j",
"name": "Spruce app notifications",
"status": "operational",
"created_at": "2021-12-10T13:25:35.080-08:00",
"updated_at": "2025-10-20T14:35:37.106-07:00",
"position": 8,
"description": "Real time notifications for new messages and video calls.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xdrmpzwhxky9",
"name": "Integrations",
"status": "operational",
"created_at": "2022-10-12T08:39:52.774-07:00",
"updated_at": "2025-10-20T12:50:00.997-07:00",
"position": 9,
"description": "This component comprises all native integrations between Spruce and external systems, such as Spruce-Elation, Spruce-Hint and Spruce-Stripe integrations.",
"showcase": true,
"start_date": "2022-10-12",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "yvdsvc2s20x7",
"name": "API",
"status": "operational",
"created_at": "2023-01-05T12:37:47.464-08:00",
"updated_at": "2025-10-20T12:50:01.014-07:00",
"position": 10,
"description": "https://api.sprucehealth.com",
"showcase": true,
"start_date": "2023-01-05",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "34nwwpqsbvfd",
"name": "Call transfer to extension failures",
"status": "resolved",
"created_at": "2025-06-18T08:11:49.995-07:00",
"updated_at": "2025-06-18T12:50:41.485-07:00",
"monitoring_at": "2025-06-18T08:11:50.000-07:00",
"resolved_at": "2025-06-18T12:50:41.466-07:00",
"impact": "none",
"shortlink": "https://stspg.io/rrfsg060qlpd",
"started_at": "2025-06-18T08:11:49.988-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "phh5vp2yrbml",
"status": "resolved",
"body": "This issue has been resolved.",
"incident_id": "34nwwpqsbvfd",
"created_at": "2025-06-18T12:50:41.466-07:00",
"updated_at": "2025-06-18T12:50:41.466-07:00",
"display_at": "2025-06-18T12:50:41.466-07:00",
"affected_components": null,
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "fd4s7vwjljh2",
"status": "monitoring",
"body": "We have identified and fixed an issue with transferring calls to internal extensions where the call would be dropped. The call transfer failures began around 4:50 A.M. PDT and resolved around 7:50 A.M. PDT. These errors only occurred for transfers to extensions. Transfers to full 10 digit numbers continued to work properly. We are continuing to monitor the situation and expect transfers to extensions to be working correctly now.",
"incident_id": "34nwwpqsbvfd",
"created_at": "2025-06-18T08:11:50.038-07:00",
"updated_at": "2025-06-18T08:12:41.352-07:00",
"display_at": "2025-06-18T08:11:50.000-07:00",
"affected_components": null,
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [],
"reminder_intervals": null
},
{
"id": "3jr2wp86ggng",
"name": "Delivery Failures for Outbound SMS, Verification Codes \u0026 Invites",
"status": "resolved",
"created_at": "2025-06-05T09:16:32.344-07:00",
"updated_at": "2025-06-05T12:27:12.724-07:00",
"monitoring_at": "2025-06-05T11:00:57.457-07:00",
"resolved_at": "2025-06-05T12:27:12.707-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/shqlp0nksm2w",
"started_at": "2025-06-05T09:16:32.338-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "7463vk0pl08l",
"status": "resolved",
"body": "The issue has been fully resolved, and we are no longer observing any SMS message delivery failures. Impacted message delivery occurred between 4:00 AM PT and approximately 11:00 AM PT on June 5, 2025.\n\nAll systems are now operating normally.\n\nWe appreciate your patience and understanding.",
"incident_id": "3jr2wp86ggng",
"created_at": "2025-06-05T12:27:12.707-07:00",
"updated_at": "2025-06-05T12:27:12.707-07:00",
"display_at": "2025-06-05T12:27:12.707-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "988pz0rgfvrz",
"status": "monitoring",
"body": "SMS delivery failures are down and our telephony partner is working on resolving any final issues. We continue to monitor the situation and will update when full resolved.",
"incident_id": "3jr2wp86ggng",
"created_at": "2025-06-05T11:00:57.457-07:00",
"updated_at": "2025-06-05T11:00:57.457-07:00",
"display_at": "2025-06-05T11:00:57.457-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "q9qcq42wp1ln",
"status": "identified",
"body": "Our telephony infrastructure partner has identified this to be a platform wide issue on their end. That being said, we are seeing a reduced rate of SMS delivery failures in the last hour. We're continuing to monitor the incident and are in active collaboration with our telephony partner to ensure that SMS delivery failures are under the expected limits.",
"incident_id": "3jr2wp86ggng",
"created_at": "2025-06-05T09:54:06.807-07:00",
"updated_at": "2025-06-05T09:54:06.807-07:00",
"display_at": "2025-06-05T09:54:06.807-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "83qpgmm56cz4",
"status": "investigating",
"body": "As of 4:00 AM PT on June 5, 2025, we began observing issues with outbound SMS delivery affecting a subset of customers. This disruption is also impacting the delivery of verification codes required for account creation and login, and may be preventing the successful sending of secure messaging invites.\n\nOur engineering team is actively investigating the root cause. We will share updates here as we learn more.",
"incident_id": "3jr2wp86ggng",
"created_at": "2025-06-05T09:16:32.449-07:00",
"updated_at": "2025-06-05T09:16:32.449-07:00",
"display_at": "2025-06-05T09:16:32.449-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "cfsmrm292xh2",
"name": "[Resolved] SMS Delivery Failures from AT\u0026T Network to a subset of Spruce Phone Numbers",
"status": "resolved",
"created_at": "2025-03-28T08:45:54.000-07:00",
"updated_at": "2025-04-01T07:41:25.208-07:00",
"monitoring_at": "2025-03-31T14:02:35.924-07:00",
"resolved_at": "2025-04-01T07:41:02.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/3d01ggzsk502",
"started_at": "2025-03-28T08:45:54.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "f72l298n1752",
"status": "resolved",
"body": "The system is operating as normal. Our telecommunications infrastructure partner has informed us that the issue with inbound SMS messages from the AT\u0026T Network to a subset of Spruce phone numbers has been resolved. It began March 28, 2025 around 8:45 PT and was resolved March 31, 2025 around 1:00 PT. We will continue to monitor.",
"incident_id": "cfsmrm292xh2",
"created_at": "2025-04-01T07:41:02.522-07:00",
"updated_at": "2025-04-01T07:41:25.194-07:00",
"display_at": "2025-04-01T07:41:02.000-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "fql98cpr5v40",
"status": "monitoring",
"body": "We are observing successful SMS delivery when sending messages from the AT\u0026T Network. We will continue to monitor to ensure full service recovery.",
"incident_id": "cfsmrm292xh2",
"created_at": "2025-03-31T14:02:35.924-07:00",
"updated_at": "2025-03-31T14:02:35.924-07:00",
"display_at": "2025-03-31T14:02:35.924-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "8szq8s36ctnd",
"status": "identified",
"body": "The issue has been acknowledged by our telecommunications infrastructure partner and they are actively collaborating with AT\u0026T on resolving this issue.",
"incident_id": "cfsmrm292xh2",
"created_at": "2025-03-31T09:20:59.592-07:00",
"updated_at": "2025-03-31T09:20:59.592-07:00",
"display_at": "2025-03-31T09:20:59.592-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "pct8csmrbd1j",
"status": "investigating",
"body": "Our telecommunications infrastructure partner has confirmed an issue in receiving SMS from AT\u0026T mobile users in some cases. They are actively collaborating with AT\u0026T to resolve this issue. This issue started on March 28, 2025 around 8:45 PT and is ongoing.",
"incident_id": "cfsmrm292xh2",
"created_at": "2025-03-31T09:19:54.199-07:00",
"updated_at": "2025-03-31T09:20:06.195-07:00",
"display_at": "2025-03-28T08:45:54.000-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "ckwmmbf00clc",
"name": "Intermittent call failures",
"status": "resolved",
"created_at": "2025-03-31T08:10:49.000-07:00",
"updated_at": "2025-03-31T12:09:42.059-07:00",
"monitoring_at": "2025-03-31T10:31:27.007-07:00",
"resolved_at": "2025-03-31T12:09:42.043-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/0gk9d5859ny5",
"started_at": "2025-03-31T08:10:49.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "696sk0lxzgln",
"status": "resolved",
"body": "Our partner has resolved the issue. All inbound and outbound phone calls are expected to operate normally.",
"incident_id": "ckwmmbf00clc",
"created_at": "2025-03-31T12:09:42.043-07:00",
"updated_at": "2025-03-31T12:09:42.043-07:00",
"display_at": "2025-03-31T12:09:42.043-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "zxr5zj0ttpks",
"status": "monitoring",
"body": "Our telecommunications provider has implemented a fix, and we started seeing improvements around 9:45am PT. We are continuing to monitor the situation.",
"incident_id": "ckwmmbf00clc",
"created_at": "2025-03-31T10:31:27.007-07:00",
"updated_at": "2025-03-31T10:31:27.007-07:00",
"display_at": "2025-03-31T10:31:27.007-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "8wycywhvwdl9",
"status": "identified",
"body": "Our telecommunications partner is experiencing issues with call routing. This has resulted in the inability to make or receive calls for some Spruce users. This issue started on March 31, 2025 around 8:10 A.M. PT and is ongoing.",
"incident_id": "ckwmmbf00clc",
"created_at": "2025-03-31T09:52:49.802-07:00",
"updated_at": "2025-03-31T10:24:38.086-07:00",
"display_at": "2025-03-31T08:10:49.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "wd35k3t198pc",
"name": "[RESOLVED] Spruce inbox inaccessible; inbound calls impacted",
"status": "resolved",
"created_at": "2025-03-11T11:48:30.000-07:00",
"updated_at": "2025-03-14T11:54:35.951-07:00",
"monitoring_at": "2025-03-11T12:18:11.983-07:00",
"resolved_at": "2025-03-11T15:20:40.360-07:00",
"impact": "critical",
"shortlink": "https://stspg.io/j11msv2qshrj",
"started_at": "2025-03-11T11:48:30.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "bgwnw64js5c2",
"status": "resolved",
"body": "The Spruce application was inaccessible across desktop, web, iOS, and Android from 11:48 AM PT to 12:02 PM PT on March 11, 2025 due to an unexpected database issue.\n\nUser Impact during this time:\n- The Spruce Inbox was unavailable to patients and providers\n- Outbound calls, SMS, video calls, and secure messages failed\n- Inbound calls failed with an error message to the caller. Most call events were delivered to the Inbox with a 10-15 minute delay, but a few were not and are being manually addressed\n- Inbound SMS and faxes were delayed by a 10-15 minutes\n\nWe are working with our cloud infrastructure provider to determine the root cause of the database issue and exploring ways to prevent similar incidents in the future.",
"incident_id": "wd35k3t198pc",
"created_at": "2025-03-11T15:20:40.360-07:00",
"updated_at": "2025-03-11T15:20:40.360-07:00",
"display_at": "2025-03-11T15:20:40.360-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "2dkwtwd0c154",
"status": "monitoring",
"body": "The platform has recovered and we are continuing to monitor.",
"incident_id": "wd35k3t198pc",
"created_at": "2025-03-11T12:18:11.983-07:00",
"updated_at": "2025-03-11T12:18:11.983-07:00",
"display_at": "2025-03-11T12:18:11.983-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "9vcnnscp6kv0",
"status": "investigating",
"body": "We are investigating an issue with unavailability of the Spruce platform",
"incident_id": "wd35k3t198pc",
"created_at": "2025-03-11T12:00:30.958-07:00",
"updated_at": "2025-03-11T15:16:47.907-07:00",
"display_at": "2025-03-11T11:48:30.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "yvdsvc2s20x7",
"name": "API",
"status": "operational",
"created_at": "2023-01-05T12:37:47.464-08:00",
"updated_at": "2025-10-20T12:50:01.014-07:00",
"position": 10,
"description": "https://api.sprucehealth.com",
"showcase": true,
"start_date": "2023-01-05",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "46q5wsd4tbj3",
"name": "[RESOLVED] Users unable to reach Spruce phone numbers with area code 317",
"status": "resolved",
"created_at": "2025-02-13T12:00:34.000-08:00",
"updated_at": "2025-02-19T07:49:57.702-08:00",
"monitoring_at": "2025-02-18T16:28:19.857-08:00",
"resolved_at": "2025-02-19T07:49:57.686-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/yprrynnl50s8",
"started_at": "2025-02-13T12:00:34.000-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "3nr74nnh2y37",
"status": "resolved",
"body": "We have more reports coming in of successful call routing to Spruce phone numbers with 317 area code. We're considering this incident resolved. If you continue to experience issues with callers unable to reach your 317 area code Spruce phone numbers, please don't hesitate to reach out to us at support@sprucehealth.com or via the Spruce Support conversation in-app.",
"incident_id": "46q5wsd4tbj3",
"created_at": "2025-02-19T07:49:57.686-08:00",
"updated_at": "2025-02-19T07:49:57.686-08:00",
"display_at": "2025-02-19T07:49:57.686-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "pz28lbvdtw0n",
"status": "monitoring",
"body": "We've been working closely with our telecommunication infrastructure provider to get this issue resolved. As of 1:35pm PT today (February 18), we have started to see recovery with multiple practices confirming that calls that were previously are successfully routing to their Spruce phone number. \n\nWe're continuing to monitor the situation to ensure it is fully resolved before considering this incident closed. We're also working with our telecommunications partner to understand the true root cause of the issue here.",
"incident_id": "46q5wsd4tbj3",
"created_at": "2025-02-18T16:28:19.857-08:00",
"updated_at": "2025-02-18T16:28:19.857-08:00",
"display_at": "2025-02-18T16:28:19.857-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "lvt6jmxy0dhf",
"status": "identified",
"body": "We're continuing to work with our telecommunications partner to get to the bottom of the call issues here. We'll provide an update tomorrow (February 18) once we have an update from them.",
"incident_id": "46q5wsd4tbj3",
"created_at": "2025-02-17T18:04:17.961-08:00",
"updated_at": "2025-02-17T18:04:17.961-08:00",
"display_at": "2025-02-17T18:04:17.961-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "dzsgr60rqh4p",
"status": "investigating",
"body": "We have been receiving multiple reports of users unable to reach Spruce phone numbers with area code 317. When dialing a 317-based Spruce phone number, the caller hears an automated message such as \"Your call cannot be completed at this time.\" We received the first report of this issue on February 13. \n\nThe impact of this issue is as follows:\n- Some providers are unable to make outbound calls via their Spruce phone number\n- Some patients are unable to reach Spruce phone numbers with are a code 317\n\nIf you're experiencing any such outbound/inbound call issues, please don't hesitate to reach out in-app via Spruce Support or email us at support@sprucehealth.com\n\nWe're actively working with our telecommunication infrastructure provider to get to bottom of this issue.",
"incident_id": "46q5wsd4tbj3",
"created_at": "2025-02-17T11:53:34.499-08:00",
"updated_at": "2025-02-17T11:54:26.338-08:00",
"display_at": "2025-02-13T12:00:34.000-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "p9vhnvndybfp",
"name": "Contacts not loading, search functional",
"status": "resolved",
"created_at": "2025-02-02T09:36:42.443-08:00",
"updated_at": "2025-02-03T04:26:31.108-08:00",
"monitoring_at": null,
"resolved_at": "2025-02-03T04:26:31.091-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/s34kg7j0bjdl",
"started_at": "2025-02-02T09:36:42.437-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "l27jkfqrdpx7",
"status": "resolved",
"body": "The issue preventing contact lists from loading for providers on Spruce has been resolved. Contacts are now loading as expected, and search remains fully functional. We appreciate your patience while we worked on the fix. If you continue to experience issues with contacts loading, please don't hesitate to reach out over the Support thread in-app or via email at support@sprucehealth.com.",
"incident_id": "p9vhnvndybfp",
"created_at": "2025-02-03T04:26:31.091-08:00",
"updated_at": "2025-02-03T04:26:31.091-08:00",
"display_at": "2025-02-03T04:26:31.091-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "h99zdyx4q5yz",
"status": "identified",
"body": "We have identified an issue where contact lists are not loading for providers on Spruce. You can still search for all your contacts. We are working to resolve the issue but it will take us a few hours to put a fix in place. We will update this page as soon as we have a fix in place and have confirmed that contacts load.",
"incident_id": "p9vhnvndybfp",
"created_at": "2025-02-02T09:36:42.534-08:00",
"updated_at": "2025-02-02T09:36:42.534-08:00",
"display_at": "2025-02-02T09:36:42.534-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "nvww789sd4pf",
"name": "[RESOLVED] SMS Deliverability issues impacting outbound SMS and verification codes",
"status": "resolved",
"created_at": "2025-01-13T13:48:58.631-08:00",
"updated_at": "2025-01-13T16:12:03.455-08:00",
"monitoring_at": "2025-01-13T15:13:46.513-08:00",
"resolved_at": "2025-01-13T16:12:03.437-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/nr46pjgmp0k8",
"started_at": "2025-01-13T13:48:58.626-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "f9g4lm1887df",
"status": "resolved",
"body": "If you continue to experience deliverability issues with SMS, please don't hesitate to reach out over the Support thread in-app or via email at support@sprucehealth.com.",
"incident_id": "nvww789sd4pf",
"created_at": "2025-01-13T16:12:03.437-08:00",
"updated_at": "2025-01-13T16:12:03.437-08:00",
"display_at": "2025-01-13T16:12:03.437-08:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "ljfwdfckh8lg",
"status": "monitoring",
"body": "The issue here was with our telecommunications infrastructure partner that was experiencing delivery failures to a subset of phone numbers across multiple networks. They have reported that the issue is now resolved and they are seeing successful delivery of SMS messages to networks.\n\nWe're continuing to work closely with them to ensure that the issue is truly resolved. If you continue to experience deliverability issues with SMS, please don't hesitate to reach out over the Support thread in-app or via email at support@sprucehealth.com.",
"incident_id": "nvww789sd4pf",
"created_at": "2025-01-13T15:13:46.513-08:00",
"updated_at": "2025-01-13T15:13:46.513-08:00",
"display_at": "2025-01-13T15:13:46.513-08:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "b1144b10sn0z",
"status": "investigating",
"body": "We are investigating an issue with outbound SMS, including verification codes. We will keep this ticket up to date as we learn more.",
"incident_id": "nvww789sd4pf",
"created_at": "2025-01-13T13:48:58.688-08:00",
"updated_at": "2025-01-13T13:48:58.688-08:00",
"display_at": "2025-01-13T13:48:58.688-08:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "rg85dwsbwp9b",
"name": "[RESOLVED] Calls and SMS impacted",
"status": "resolved",
"created_at": "2024-11-04T10:24:38.285-08:00",
"updated_at": "2024-11-04T13:35:16.952-08:00",
"monitoring_at": "2024-11-04T10:41:21.673-08:00",
"resolved_at": "2024-11-04T13:35:08.347-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/82rz3nc00x2t",
"started_at": "2024-11-04T10:24:38.276-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "xcsqmhfxtwqn",
"status": "resolved",
"body": "The system is operating as normal. This seems to have been an intermittent underlying issue with our telecom infrastructure provider which we are continuing to investigate and requesting updates on.",
"incident_id": "rg85dwsbwp9b",
"created_at": "2024-11-04T13:35:08.347-08:00",
"updated_at": "2024-11-04T13:35:08.347-08:00",
"display_at": "2024-11-04T13:35:08.347-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "zmzv09nx3391",
"status": "monitoring",
"body": "The issue was temporary and lasted from 10:12 am PT to 10:21 am PT. During this time, inbound and outbound calls were impacted for some organizations where inbound calls likely failed and a corresponding failed call events were posted. Outbound calls were also likely impacted for some organizations. Any inbound/outbound SMS, if delayed were eventually processed.\n\nWe are continuing to monitor the situation to ensure that the platform is functioning normally. We are also continuing to investigate what caused the issue and how we can improve it for the future.",
"incident_id": "rg85dwsbwp9b",
"created_at": "2024-11-04T10:41:21.673-08:00",
"updated_at": "2024-11-04T10:41:21.673-08:00",
"display_at": "2024-11-04T10:41:21.673-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "xhw8lwmhjrwh",
"status": "investigating",
"body": "As of 10:12am PT, we are seeing calls and SMS impacted where inbound and outbound calls may be failing and SMS may be delayed. We're actively investigating this issue.",
"incident_id": "rg85dwsbwp9b",
"created_at": "2024-11-04T10:24:38.415-08:00",
"updated_at": "2024-11-04T10:24:38.415-08:00",
"display_at": "2024-11-04T10:24:38.415-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "1krfnx0vyhhx",
"name": "Stale search results when searching for contacts, conversations or messages",
"status": "resolved",
"created_at": "2024-10-24T12:00:13.000-07:00",
"updated_at": "2024-10-25T09:32:04.000-07:00",
"monitoring_at": null,
"resolved_at": "2024-10-25T09:32:03.978-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/l02t9zpvgdhh",
"started_at": "2024-10-24T12:00:13.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "qr8bbppy8hvn",
"status": "resolved",
"body": "Search latency is back to a normal state as of 2am PT / 5 am ET October 25, 2024. We have a good understanding of what happened here and are looking into ways we can reduce the likelihood of a similar situation in the future. \n\nCustomer impact was from ~12pm PT / 3pm ET on October 24, 2024 to 2am PT / 5 am ET where search results were stale for conversation, contact and contact list querying.\n\nThere was no impact to the Spruce inbox, exchanging of calls, SMS, secure messaging, email, video calls or fax on the platform.",
"incident_id": "1krfnx0vyhhx",
"created_at": "2024-10-25T09:32:03.978-07:00",
"updated_at": "2024-10-25T09:32:03.978-07:00",
"display_at": "2024-10-25T09:32:03.978-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "f01flrmdrss8",
"status": "identified",
"body": "We have identified the issue and are working through a plan to fully resolve the issue. The latency for updates in the system to show up in the search results is ~15 minutes. We're working to see how we can improve this and fully resolve the issue.",
"incident_id": "1krfnx0vyhhx",
"created_at": "2024-10-24T17:47:09.651-07:00",
"updated_at": "2024-10-24T17:47:09.651-07:00",
"display_at": "2024-10-24T17:47:09.651-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "8p2j6zcjqxhm",
"status": "investigating",
"body": "We're seeing elevated latencies in indexing contacts, conversations and messages for the purpose of searching. This is resulting in potentially stale search results being returned when searching for contacts, conversations and messages or even when loading contact lists.\n\nThis issue started around 12pm PT / 3pm ET and is ongoing. We're actively investigating the issue and will post updates here as soon as we have them.\n\nThere is no impact to loading the Spruce inbox, making/receiving calls, exchanging SMS, email or fax on Spruce.",
"incident_id": "1krfnx0vyhhx",
"created_at": "2024-10-24T15:44:13.249-07:00",
"updated_at": "2024-10-24T15:44:23.368-07:00",
"display_at": "2024-10-24T12:00:13.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "z5t2jghycwv2",
"name": "Spruce experiencing issues with calls, SMS and loading inbox.",
"status": "resolved",
"created_at": "2024-09-25T09:41:59.000-07:00",
"updated_at": "2024-09-25T10:44:11.533-07:00",
"monitoring_at": "2024-09-25T09:41:59.000-07:00",
"resolved_at": "2024-09-25T10:36:05.405-07:00",
"impact": "major",
"shortlink": "https://stspg.io/b5wkzvt4vy6w",
"started_at": "2024-09-25T09:41:59.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "f3w6ddshz171",
"status": "resolved",
"body": "This incident has been resolved.",
"incident_id": "z5t2jghycwv2",
"created_at": "2024-09-25T10:36:05.405-07:00",
"updated_at": "2024-09-25T10:36:05.405-07:00",
"display_at": "2024-09-25T10:36:05.405-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "partial_outage",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "partial_outage",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "partial_outage",
"new_status": "operational"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "partial_outage",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "partial_outage",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "partial_outage",
"new_status": "operational"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "partial_outage",
"new_status": "operational"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "partial_outage",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "xwdtyg05y1xk",
"status": "monitoring",
"body": "This issue has since been resolved but we are monitoring the situation as everything returns to normal.",
"incident_id": "z5t2jghycwv2",
"created_at": "2024-09-25T10:27:25.830-07:00",
"updated_at": "2024-09-25T10:44:11.528-07:00",
"display_at": "2024-09-25T10:04:25.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "partial_outage",
"new_status": "partial_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "partial_outage",
"new_status": "partial_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "partial_outage",
"new_status": "partial_outage"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "partial_outage",
"new_status": "partial_outage"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "partial_outage",
"new_status": "partial_outage"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "partial_outage",
"new_status": "partial_outage"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "partial_outage",
"new_status": "partial_outage"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "partial_outage",
"new_status": "partial_outage"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "270x4xjwdm7w",
"status": "monitoring",
"body": "We identified an issue causing unavailability of SMS, calls, fax and loading of the inbox. We are investigating a resolution.",
"incident_id": "z5t2jghycwv2",
"created_at": "2024-09-25T10:25:59.927-07:00",
"updated_at": "2024-09-25T10:30:22.773-07:00",
"display_at": "2024-09-25T09:41:59.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "partial_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "partial_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "partial_outage"
},
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "partial_outage"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "partial_outage"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "partial_outage"
},
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "partial_outage"
},
{
"code": "yvdsvc2s20x7",
"name": "API",
"old_status": "operational",
"new_status": "partial_outage"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "6wxpvnqpkbsg",
"name": "Fax",
"status": "operational",
"created_at": "2018-08-14T18:45:20.310-07:00",
"updated_at": "2025-10-28T11:58:43.083-07:00",
"position": 5,
"description": "Inbound/Outbound Digital Fax",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xdrmpzwhxky9",
"name": "Integrations",
"status": "operational",
"created_at": "2022-10-12T08:39:52.774-07:00",
"updated_at": "2025-10-20T12:50:00.997-07:00",
"position": 9,
"description": "This component comprises all native integrations between Spruce and external systems, such as Spruce-Elation, Spruce-Hint and Spruce-Stripe integrations.",
"showcase": true,
"start_date": "2022-10-12",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "yvdsvc2s20x7",
"name": "API",
"status": "operational",
"created_at": "2023-01-05T12:37:47.464-08:00",
"updated_at": "2025-10-20T12:50:01.014-07:00",
"position": 10,
"description": "https://api.sprucehealth.com",
"showcase": true,
"start_date": "2023-01-05",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "tv3yvqb7jqny",
"name": "[RESOLVED] Messages delayed to Spruce Inbox",
"status": "resolved",
"created_at": "2024-09-03T07:20:35.000-07:00",
"updated_at": "2024-09-03T11:17:37.974-07:00",
"monitoring_at": "2024-09-03T10:53:22.747-07:00",
"resolved_at": "2024-09-03T11:17:29.450-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/xk3h15y071lg",
"started_at": "2024-09-03T07:20:35.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "vbw4stfmf9v1",
"status": "resolved",
"body": "The system has recovered after the increase capacity. There should no longer be a delay in messages showing up in the Spruce inbox. The customer impact was from 07:20 PT - 10:40 PT on Sept 3 2024.",
"incident_id": "tv3yvqb7jqny",
"created_at": "2024-09-03T11:17:29.450-07:00",
"updated_at": "2024-09-03T11:17:29.450-07:00",
"display_at": "2024-09-03T11:17:29.450-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "7fhqvzk3lsjp",
"status": "monitoring",
"body": "We have increased the capacity of the system to reduce the delay in messages showing up in the Spruce inbox. The increased capacity has resulted in recovery where messages are now showing up in the inbox in a timely fashion.\n\nWe're continuing to monitor the system to ensure normal operations while also leaving the increased capacity in place.",
"incident_id": "tv3yvqb7jqny",
"created_at": "2024-09-03T10:53:22.747-07:00",
"updated_at": "2024-09-03T10:53:22.747-07:00",
"display_at": "2024-09-03T10:53:22.747-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "mfggm0ym8f76",
"status": "investigating",
"body": "Some providers and patients may be experiencing a delay in messages showing up in the Spruce inbox. There is no impact to making/receiving calls, video calls, or sending messages.",
"incident_id": "tv3yvqb7jqny",
"created_at": "2024-09-03T10:25:35.854-07:00",
"updated_at": "2024-09-03T10:25:58.445-07:00",
"display_at": "2024-09-03T07:20:35.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "s7s4856zpdpb",
"name": "[RESOLVED] Call transfers from VoIP desk phones not working",
"status": "resolved",
"created_at": "2024-08-07T07:12:10.000-07:00",
"updated_at": "2024-08-07T15:21:19.737-07:00",
"monitoring_at": null,
"resolved_at": "2024-08-07T08:07:03.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/4kmy85ht5qd7",
"started_at": "2024-08-07T07:12:10.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "x9xqj8cxp2g8",
"status": "resolved",
"body": "The root cause of this outage was the deployment of new code that attempted to manage the transferring of calls to international numbers. The code incorrectly made the assumption that the “to” identifier for the transfer was always a phone number and attempted to parse it as such. In reality this “to” information can also be SIP and Client identifiers. Failing to parse the numbers and treating that failure as a hard failure for the call resulted in the transfer flows failing.",
"incident_id": "s7s4856zpdpb",
"created_at": "2024-08-07T15:20:03.230-07:00",
"updated_at": "2024-08-07T15:21:19.721-07:00",
"display_at": "2024-08-07T08:07:03.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "n3h0qk744bwt",
"status": "identified",
"body": "From 5:03 AM PT to 7:56 AM PT on August 7, 2024, call transfers from VoIP desk phones failed to complete and resulted in an error being reported to the user initiating the call transfer.\n\nNo other inbound/outbound calls were impacted during this time. The rest of the platform continued to operate as intended.",
"incident_id": "s7s4856zpdpb",
"created_at": "2024-08-07T15:19:10.582-07:00",
"updated_at": "2024-08-07T15:21:14.248-07:00",
"display_at": "2024-08-07T07:12:10.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "2z92s6fkgrzp",
"name": "Spruce inbox failing to load",
"status": "resolved",
"created_at": "2024-05-01T11:47:39.824-07:00",
"updated_at": "2024-05-01T12:26:14.794-07:00",
"monitoring_at": "2024-05-01T12:01:42.171-07:00",
"resolved_at": "2024-05-01T12:26:14.780-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/z39vdwr7r1ys",
"started_at": "2024-05-01T11:47:39.818-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "h3s5sf24lr8f",
"status": "resolved",
"body": "The system is operating normally and the inbox is accessible now. There was no impact to inbound calls, fax or SMS. As the Spruce app was inaccessible, patients and providers likely could not send a message, engage in a video call or make an outbound call.",
"incident_id": "2z92s6fkgrzp",
"created_at": "2024-05-01T12:26:14.780-07:00",
"updated_at": "2024-05-01T12:26:14.780-07:00",
"display_at": "2024-05-01T12:26:14.780-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "8dvycnkxc946",
"status": "monitoring",
"body": "The system is operating normally and the inbox is accessible now. There was no impact to inbound calls, fax or SMS. As the Spruce app was inaccessible, patients and providers likely could not send a message, engage in a video call or make an outbound call. \n\nImpact was from 11:39am PT - 11:51am PT on May 1 2024.",
"incident_id": "2z92s6fkgrzp",
"created_at": "2024-05-01T12:26:04.361-07:00",
"updated_at": "2024-05-01T12:26:04.361-07:00",
"display_at": "2024-05-01T12:26:04.361-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "kxnzpzytz1xg",
"status": "monitoring",
"body": "We started to see recovery as of 11:50am PT where the inbox was accessible for users. We're continuing to monitor the situation to ensure that the platform remains operational.",
"incident_id": "2z92s6fkgrzp",
"created_at": "2024-05-01T12:01:42.171-07:00",
"updated_at": "2024-05-01T12:01:42.171-07:00",
"display_at": "2024-05-01T12:01:42.171-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "7525374hs6hc",
"status": "identified",
"body": "We have identified the issue and are working to resolve the issue as we speak. Inbound calls are not impacted. Outbound calls may be impacted.",
"incident_id": "2z92s6fkgrzp",
"created_at": "2024-05-01T11:52:32.882-07:00",
"updated_at": "2024-05-01T11:52:32.882-07:00",
"display_at": "2024-05-01T11:52:32.882-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "s2nwf6t50jy9",
"status": "investigating",
"body": "We are investigating an issue with the Spruce inbox failing to load.",
"incident_id": "2z92s6fkgrzp",
"created_at": "2024-05-01T11:47:39.870-07:00",
"updated_at": "2024-05-01T11:47:39.870-07:00",
"display_at": "2024-05-01T11:47:39.870-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "fx0n97w8nf7d",
"name": "Outbound and Inbound calls not connecting on Spruce",
"status": "resolved",
"created_at": "2024-04-22T07:55:56.875-07:00",
"updated_at": "2024-04-22T11:32:46.762-07:00",
"monitoring_at": "2024-04-22T08:55:20.946-07:00",
"resolved_at": "2024-04-22T11:32:46.746-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/byz00py5j268",
"started_at": "2024-04-22T07:55:56.870-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "y0xyg92gcgq1",
"status": "resolved",
"body": "Our telecommunications infrastructure provider has confirmed that the system is fully operational now. Inbound and outbound calls for many customers were impacted between 7:15am PT and 10:30 am PT on April 22, 2024. All call events, including any voicemails, should be in the Spruce inbox.\n\nWe apologize for the disruption this issue may have caused to your practice operations. We're working closely with our telecommunications provider to identify the root cause of the issue and engage with them in ways to reduce the likelihood of such an issue from occurring again.",
"incident_id": "fx0n97w8nf7d",
"created_at": "2024-04-22T11:32:46.746-07:00",
"updated_at": "2024-04-22T11:32:46.746-07:00",
"display_at": "2024-04-22T11:32:46.746-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "zzh5tm8dq9b2",
"status": "monitoring",
"body": "Our telecommunications infrastructure provider identified the problem to be a scaling issue. They have scaled their resources appropriately and are starting to see recovery on their end. In our testing, inbound and outbound calls seem to be working now. It will likely take a few more minutes for inbound and outbound calls to work for all customers. We'll continue to monitor the situation and post an update here.",
"incident_id": "fx0n97w8nf7d",
"created_at": "2024-04-22T08:55:20.946-07:00",
"updated_at": "2024-04-22T08:55:20.946-07:00",
"display_at": "2024-04-22T08:55:20.946-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "cr4c3xqk8j50",
"status": "investigating",
"body": "Our telecommunications infrastructure provider has acknowledged that this is an issue on their end. Their engineering team is actively looking into the issue. We are in close contact with them and will continue to update this incident page as soon as we have another update to share.",
"incident_id": "fx0n97w8nf7d",
"created_at": "2024-04-22T08:04:01.822-07:00",
"updated_at": "2024-04-22T08:04:01.822-07:00",
"display_at": "2024-04-22T08:04:01.822-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "0pxgfzg06g4y",
"status": "investigating",
"body": "We are receiving multiple reports of practices being unable to hear the other party on inbound and outbound calls. We are actively investigating the issue and will keep this incident page updated as we have updates to share.",
"incident_id": "fx0n97w8nf7d",
"created_at": "2024-04-22T07:55:56.948-07:00",
"updated_at": "2024-04-22T07:55:56.948-07:00",
"display_at": "2024-04-22T07:55:56.948-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "ysjnt0wd0f89",
"name": "Users on Comcast/Xfinity reporting issues connecting to Spruce",
"status": "resolved",
"created_at": "2024-03-05T12:59:17.787-08:00",
"updated_at": "2024-03-05T14:03:49.445-08:00",
"monitoring_at": null,
"resolved_at": "2024-03-05T14:03:49.401-08:00",
"impact": "none",
"shortlink": "https://stspg.io/l2ntv4xln4df",
"started_at": "2024-03-05T12:59:17.781-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "4m7yzj79nr5m",
"status": "resolved",
"body": "The external network connectivity issues have been resolved. The service should now be fully accessible for everyone.",
"incident_id": "ysjnt0wd0f89",
"created_at": "2024-03-05T14:03:49.401-08:00",
"updated_at": "2024-03-05T14:03:49.401-08:00",
"display_at": "2024-03-05T14:03:49.401-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "6t9wj04pbcz6",
"status": "investigating",
"body": "We are receiving reports of users on Comcast/Xfinity's internet service being unable to connect to Spruce. The Spruce platform remains operational and this seems to isolated to users on Comcast/Xfinity. \n\nIf you are on Comcast/Xfinity, please try connecting through a different internet service provider to reach Spruce. You could try the mobile apps via a cellular network connection in the mean time.\n\nWe are actively monitoring the situation and ensuring that the impact is not more widespread. We are also looking for an official report on this to ensure that our assessment is correct.",
"incident_id": "ysjnt0wd0f89",
"created_at": "2024-03-05T12:59:17.832-08:00",
"updated_at": "2024-03-05T12:59:17.832-08:00",
"display_at": "2024-03-05T12:59:17.832-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "kks0b7ld3xdl",
"name": "AT\u0026T outage impacting calls and SMS on Spruce",
"status": "resolved",
"created_at": "2024-02-22T07:39:15.194-08:00",
"updated_at": "2024-02-22T13:00:04.554-08:00",
"monitoring_at": null,
"resolved_at": "2024-02-22T13:00:04.541-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/kmxkcq074vs8",
"started_at": "2024-02-22T07:39:15.187-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "100hh2yr94lm",
"status": "resolved",
"body": "AT\u0026T is reporting that, as of 3pm ET, service has been restored to all impacted users. Spruce services should be operating normally for patients and providers at this point. If you are experiencing any issues, please don't hesitate to reach us at support@sprucehealth.com.",
"incident_id": "kks0b7ld3xdl",
"created_at": "2024-02-22T13:00:04.541-08:00",
"updated_at": "2024-02-22T13:00:04.541-08:00",
"display_at": "2024-02-22T13:00:04.541-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "fqk34pkdmrzr",
"status": "identified",
"body": "AT\u0026T is currently experiencing an outage which is having the following impact.\n\nProviders with AT\u0026T as their cell phone service provider may experience the following:\n- Unable to make calls from the Spruce application\n- Unable to receive verification codes to log in to Spruce\n\nAll providers may experience the following:\n- Callers using AT\u0026T unable to call Spruce phone numbers\n- Texters using AT\u0026T unable to SMS Spruce phone numbers\n- Providers calling patients with AT\u0026T unable to receive calls\n\nThe following services are NOT impacted:\n- Exchanging secure messages, fax and email via Spruce\n- Video calling on Spruce",
"incident_id": "kks0b7ld3xdl",
"created_at": "2024-02-22T07:39:15.293-08:00",
"updated_at": "2024-02-22T07:39:15.293-08:00",
"display_at": "2024-02-22T07:39:15.293-08:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "h181rq4bjm5g",
"name": "Video calls not connecting",
"status": "resolved",
"created_at": "2024-02-15T14:12:57.766-08:00",
"updated_at": "2024-02-15T17:24:41.553-08:00",
"monitoring_at": "2024-02-15T15:47:17.370-08:00",
"resolved_at": "2024-02-15T17:24:41.540-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/vy3q06s6f0jz",
"started_at": "2024-02-15T14:12:57.762-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "frwstffjny3j",
"status": "resolved",
"body": "Video calls should now be functioning on Spruce. The issue was with our underlying video infrastructure provider. We are still gathering the details on what went wrong on their end and will post more details as a postmortem to this incident page. We're sorry for the inconvenience this incident caused to your business.",
"incident_id": "h181rq4bjm5g",
"created_at": "2024-02-15T17:24:41.540-08:00",
"updated_at": "2024-02-15T17:24:41.540-08:00",
"display_at": "2024-02-15T17:24:41.540-08:00",
"affected_components": [
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "007t92b1b4nk",
"status": "monitoring",
"body": "A fix has been implemented and we are seeing the issue being resolved. We will continue to monitor to ensure video calls are fully operational.",
"incident_id": "h181rq4bjm5g",
"created_at": "2024-02-15T15:47:17.370-08:00",
"updated_at": "2024-02-15T15:47:17.370-08:00",
"display_at": "2024-02-15T15:47:17.370-08:00",
"affected_components": [
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "6qwmkpzytjrx",
"status": "investigating",
"body": "We have engaged our video infrastructure provider to help investigate. We have been able to reproduce this issue on our end. We're actively working on this issue and will update this page as soon as we have an update to share.",
"incident_id": "h181rq4bjm5g",
"created_at": "2024-02-15T15:02:07.483-08:00",
"updated_at": "2024-02-15T15:02:07.483-08:00",
"display_at": "2024-02-15T15:02:07.483-08:00",
"affected_components": [
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "76vgvlvxjd4z",
"status": "investigating",
"body": "We are investigating an issue with video calls where patients are unable to join a video call with their provider.",
"incident_id": "h181rq4bjm5g",
"created_at": "2024-02-15T14:12:57.832-08:00",
"updated_at": "2024-02-15T14:12:57.832-08:00",
"display_at": "2024-02-15T14:12:57.832-08:00",
"affected_components": [
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "m41qcbw5tssb",
"name": "[RESOLVED] Search \u0026 Contact Filters experiencing intermittent errors and slowness",
"status": "resolved",
"created_at": "2024-01-09T14:26:28.957-08:00",
"updated_at": "2024-01-09T22:08:20.299-08:00",
"monitoring_at": "2024-01-09T15:17:02.667-08:00",
"resolved_at": "2024-01-09T22:08:20.287-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/w2vn3kyx1nv1",
"started_at": "2024-01-09T14:26:28.951-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "0jp09zw626vr",
"status": "resolved",
"body": "The platform is fully operational. Customer impact was from 1:08pm PT to 3:14pm PT.",
"incident_id": "m41qcbw5tssb",
"created_at": "2024-01-09T22:08:20.287-08:00",
"updated_at": "2024-01-09T22:08:20.287-08:00",
"display_at": "2024-01-09T22:08:20.287-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "htl5841m3m89",
"status": "monitoring",
"body": "The system should be fully operational now and search results should no longer be stale. We will continue to actively monitor the platform to ensure smooth operations.",
"incident_id": "m41qcbw5tssb",
"created_at": "2024-01-09T15:17:02.667-08:00",
"updated_at": "2024-01-09T15:17:02.667-08:00",
"display_at": "2024-01-09T15:17:02.667-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "psv60th930vz",
"status": "identified",
"body": "Users may be experiencing degraded performance for the following actions:\n- Searching for contacts or conversations: may be giving errors, returning results in a delayed manner or returning stale results\n- Creating conversations: maybe return errors or experiencing a lag when searching for contacts\n- Sending bulk messages: may be experiencing a lag where stuck in Processing state for a while or just taking longer to complete\n- Opening contact lists: may be returning errors or experiencing a lag in returning results\n\nPhone calls, video calls, exchanging messages (fax, secure, email or SMS) is NOT impacted and operating normally.\n\nWe are actively working on resolving this issue and will provide an update soon.",
"incident_id": "m41qcbw5tssb",
"created_at": "2024-01-09T14:26:29.062-08:00",
"updated_at": "2024-01-09T14:26:29.062-08:00",
"display_at": "2024-01-09T14:26:29.062-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "8txdfrwvrsfm",
"name": "Delayed Sending of Outbound Messages",
"status": "resolved",
"created_at": "2023-11-13T09:57:27.000-08:00",
"updated_at": "2023-11-13T16:04:50.185-08:00",
"monitoring_at": "2023-11-13T12:06:19.098-08:00",
"resolved_at": "2023-11-13T16:04:50.172-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/k821m7x2nnnq",
"started_at": "2023-11-13T09:57:27.000-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "gqbgvy4v2gnc",
"status": "resolved",
"body": "Outbound SMS is fully functional at this point, along with the rest of the services.",
"incident_id": "8txdfrwvrsfm",
"created_at": "2023-11-13T16:04:50.172-08:00",
"updated_at": "2023-11-13T16:04:50.172-08:00",
"display_at": "2023-11-13T16:04:50.172-08:00",
"affected_components": [
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "8vgcpmswqvrd",
"status": "monitoring",
"body": "A fix has been implemented and we are monitoring the results.",
"incident_id": "8txdfrwvrsfm",
"created_at": "2023-11-13T12:06:19.098-08:00",
"updated_at": "2023-11-13T12:06:19.098-08:00",
"display_at": "2023-11-13T12:06:19.098-08:00",
"affected_components": [
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "6fzn743dc4ck",
"status": "investigating",
"body": "An issue with our outbound message processing caused SMS, fax, and email messages to be delayed during the period of this event. We have since addressed the issue and outbound messaging is behaving normally again. No other operations were impacted during this event.",
"incident_id": "8txdfrwvrsfm",
"created_at": "2023-11-13T12:05:27.457-08:00",
"updated_at": "2023-11-13T12:12:41.329-08:00",
"display_at": "2023-11-13T09:57:27.000-08:00",
"affected_components": [
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "6wxpvnqpkbsg",
"name": "Fax",
"status": "operational",
"created_at": "2018-08-14T18:45:20.310-07:00",
"updated_at": "2025-10-28T11:58:43.083-07:00",
"position": 5,
"description": "Inbound/Outbound Digital Fax",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "tsn8d72kxdlc",
"name": "[RESOLVED] Inbound SMS delayed",
"status": "resolved",
"created_at": "2023-11-07T11:23:08.000-08:00",
"updated_at": "2023-11-07T15:04:18.671-08:00",
"monitoring_at": null,
"resolved_at": "2023-11-07T15:04:18.657-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/8x0k6fzybskd",
"started_at": "2023-11-07T11:23:08.000-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "zprvhky5y10b",
"status": "resolved",
"body": "The issue has been resolved and all systems are operating normally. We're sorry for the inconvenience caused here, and are looking into ways to detect such a failure early on in the deployment process.",
"incident_id": "tsn8d72kxdlc",
"created_at": "2023-11-07T15:04:18.657-08:00",
"updated_at": "2023-11-07T15:04:18.657-08:00",
"display_at": "2023-11-07T15:04:18.657-08:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "flsb3tt9zh8m",
"status": "investigating",
"body": "All inbound SMS from 11:21am PT to 1:53pm PT on November 7 2023 was delivered to the Spruce inbox in a delayed manner. There was no impact to outbound SMS or to any other aspect of the Spruce platform. \n\nAll inbound SMS that were delayed had an indicator in the message itself to indicate how long they were delayed by (the time between when the SMS was received by our telecommunications carrier partner and when the SMS was delivered to the Spruce inbox).",
"incident_id": "tsn8d72kxdlc",
"created_at": "2023-11-07T15:03:08.317-08:00",
"updated_at": "2023-11-07T15:03:21.595-08:00",
"display_at": "2023-11-07T11:23:08.000-08:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "lhfnzzcv4y6v",
"name": "Intermittent failures",
"status": "resolved",
"created_at": "2023-10-25T09:35:15.098-07:00",
"updated_at": "2023-10-25T11:20:31.377-07:00",
"monitoring_at": null,
"resolved_at": "2023-10-25T10:40:10.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/k0mkc0n29jtn",
"started_at": "2023-10-25T09:35:15.092-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "391nzs3wcb0w",
"status": "resolved",
"body": "While deploying changes we were hitting rate limits with our underlying infrastructure provider. This was resulting in intermittent errors while using the apps, but most functionality including calls/sms/fax continued to work without any known problems. We now have higher limits to avoid this happening again in the future.",
"incident_id": "lhfnzzcv4y6v",
"created_at": "2023-10-25T11:19:10.734-07:00",
"updated_at": "2023-10-25T11:20:31.360-07:00",
"display_at": "2023-10-25T10:40:10.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "rw42wpjf1fgj",
"status": "investigating",
"body": "We are currently investigating reports of intermittent failures affecting the web and mobile apps.",
"incident_id": "lhfnzzcv4y6v",
"created_at": "2023-10-25T09:35:15.175-07:00",
"updated_at": "2023-10-25T09:35:15.175-07:00",
"display_at": "2023-10-25T09:35:15.175-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "pwbzl3r6p7m0",
"name": "Partial MMS/SMS message delivery failures",
"status": "resolved",
"created_at": "2023-09-28T12:16:57.000-07:00",
"updated_at": "2023-09-28T15:59:16.171-07:00",
"monitoring_at": "2023-09-28T12:16:57.000-07:00",
"resolved_at": "2023-09-28T15:59:16.156-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/p2t28hhlwtfq",
"started_at": "2023-09-28T12:16:57.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "39vz62k8xxhj",
"status": "resolved",
"body": "The issues involving SMS/MMS have been resolved and all messaging should be returned to normal.",
"incident_id": "pwbzl3r6p7m0",
"created_at": "2023-09-28T15:59:16.156-07:00",
"updated_at": "2023-09-28T15:59:16.156-07:00",
"display_at": "2023-09-28T15:59:16.156-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "l6qvjlrkfts0",
"status": "monitoring",
"body": "Our upstream infrastructure partner has reported that the underlying issue causing delivery failures to Verizon and AT\u0026T networks is resolving and successful sends of SMS/MMS messages should be returning to normal rates.",
"incident_id": "pwbzl3r6p7m0",
"created_at": "2023-09-28T14:11:33.426-07:00",
"updated_at": "2023-09-28T14:19:16.648-07:00",
"display_at": "2023-09-28T14:11:33.000-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "mbx9vfm3q2hd",
"status": "monitoring",
"body": "We are experiencing MMS/SMS message delivery failures to Verizon and AT\u0026T networks in the US for a subset of phone numbers, due to incorrect message filtering (blocking) by an upstream infrastructure partner. We are working on the problem and will update regularly.",
"incident_id": "pwbzl3r6p7m0",
"created_at": "2023-09-28T12:16:57.167-07:00",
"updated_at": "2023-09-28T12:25:22.222-07:00",
"display_at": "2023-09-28T12:16:57.000-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "qtwd0qvnpwk3",
"name": "Search \u0026 Contact Filters experiencing intermittent errors and slowness",
"status": "resolved",
"created_at": "2023-09-25T14:32:40.400-07:00",
"updated_at": "2023-09-25T16:48:13.161-07:00",
"monitoring_at": null,
"resolved_at": "2023-09-25T16:48:13.146-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/84g0khrdhyxf",
"started_at": "2023-09-25T14:32:40.396-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "vysbw1bftr3y",
"status": "resolved",
"body": "We identified the reason for performance degradation to be an imbalance of data spread across the cluster. We have since resolved this issue and ensured that all stale search results have been cleared.\n\nThe system is fully operational at this point.",
"incident_id": "qtwd0qvnpwk3",
"created_at": "2023-09-25T16:48:13.146-07:00",
"updated_at": "2023-09-25T16:48:13.146-07:00",
"display_at": "2023-09-25T16:48:13.146-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "n1s7j0xpq6k5",
"status": "investigating",
"body": "Users may be experiencing degraded performance for the following actions:\n- Searching for contacts or conversations: may be giving errors, returning results in a delayed manner or returning stale results\n- Creating conversations: maybe return errors or experiencing a lag when searching for contacts\n- Sending bulk messages: may be experiencing a lag where stuck in Processing state for a while or just taking longer to complete\n- Opening contact lists: may be returning errors or experiencing a lag in returning results\n\nPhone calls, video calls, exchanging messages (fax, secure, email or SMS) is NOT impacted and operating normally.",
"incident_id": "qtwd0qvnpwk3",
"created_at": "2023-09-25T14:32:40.462-07:00",
"updated_at": "2023-09-25T14:32:40.462-07:00",
"display_at": "2023-09-25T14:32:40.462-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "r3kktq81pm2r",
"name": "Issues with inbound and outbound fax delivery.",
"status": "resolved",
"created_at": "2023-09-15T11:55:51.402-07:00",
"updated_at": "2023-09-15T12:59:53.763-07:00",
"monitoring_at": null,
"resolved_at": "2023-09-15T12:59:53.746-07:00",
"impact": "critical",
"shortlink": "https://stspg.io/l0pcd8kc93d8",
"started_at": "2023-09-15T11:55:51.397-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "0vhjtpt52drk",
"status": "resolved",
"body": "The issue with faxes has been resolved.",
"incident_id": "r3kktq81pm2r",
"created_at": "2023-09-15T12:59:53.746-07:00",
"updated_at": "2023-09-15T12:59:53.746-07:00",
"display_at": "2023-09-15T12:59:53.746-07:00",
"affected_components": [
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "major_outage",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "sblghy2q4tpl",
"status": "investigating",
"body": "Our fax provider is currently experiencing issues with inbound and outbound faxes. We will continue to monitor and update as more information is available from https://status.phaxio.com",
"incident_id": "r3kktq81pm2r",
"created_at": "2023-09-15T11:55:51.479-07:00",
"updated_at": "2023-09-15T11:55:51.479-07:00",
"display_at": "2023-09-15T11:55:51.479-07:00",
"affected_components": [
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "6wxpvnqpkbsg",
"name": "Fax",
"status": "operational",
"created_at": "2018-08-14T18:45:20.310-07:00",
"updated_at": "2025-10-28T11:58:43.083-07:00",
"position": 5,
"description": "Inbound/Outbound Digital Fax",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "rd16m3h7hrv5",
"name": "Degraded SMS functionality",
"status": "resolved",
"created_at": "2023-08-17T17:39:04.000-07:00",
"updated_at": "2023-08-17T22:12:31.979-07:00",
"monitoring_at": "2023-08-17T17:52:08.478-07:00",
"resolved_at": "2023-08-17T19:50:20.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/t6qvtt139zl3",
"started_at": "2023-08-17T17:39:04.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "cd6dk5dtmbnh",
"status": "resolved",
"body": "Our telecommunications provider confirmed all systems to be operational at 7:44pm. During this incident, outbound SMS from Spruce was delayed in some instances from 5pm PT to 5:50pm PT. Users may not have been able to log in to Spruce if verification code was required between 5pm PT and 5:50pm PT. All other aspects of Spruce were functional during this time.",
"incident_id": "rd16m3h7hrv5",
"created_at": "2023-08-17T22:12:20.618-07:00",
"updated_at": "2023-08-17T22:12:31.959-07:00",
"display_at": "2023-08-17T19:50:20.000-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "gz24qw8tbr5l",
"status": "monitoring",
"body": "While we are not seeing any issues with outbound SMS, our telecommunications provider continues to report degraded functionality. So we'll continue to monitor on our end.",
"incident_id": "rd16m3h7hrv5",
"created_at": "2023-08-17T18:42:41.191-07:00",
"updated_at": "2023-08-17T18:42:41.191-07:00",
"display_at": "2023-08-17T18:42:41.191-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "dpylzkwnyb7g",
"status": "monitoring",
"body": "Outbound SMS looks all caught up at this point with no delays in sending SMS. We'll continue to monitor to see if the issue persists and wait until we have received confirmation from our telecommunications provider that SMS is functional on their end.",
"incident_id": "rd16m3h7hrv5",
"created_at": "2023-08-17T17:52:08.478-07:00",
"updated_at": "2023-08-17T17:52:08.478-07:00",
"display_at": "2023-08-17T17:52:08.478-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "45jkzr1b300z",
"status": "identified",
"body": "Users may not be able to log in to Spruce if they are waiting on the verification code since SMS functionality is degraded. \n\nOur telecommunications provider is experiencing issues with SMS functionality which in turn is impacting Spruce's ability to send SMS.",
"incident_id": "rd16m3h7hrv5",
"created_at": "2023-08-17T17:43:47.044-07:00",
"updated_at": "2023-08-17T17:43:47.044-07:00",
"display_at": "2023-08-17T17:43:47.044-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "lpghtftqrsh3",
"status": "investigating",
"body": "We are seeing elevated errors on the system when attempting to send SMS from Spruce. Our telecommunications infrastructure provider is reporting an issue in the SMS functionality which corresponds to the timeline of errors on our end. \n\nOutbound SMS is definitely impacted given that we are seeing errors. All outbound SMS that was sent from the Spruce app will eventually be delivered when our telecommunications provider's systems are operational again.\nIt is unclear if inbound SMS to Spruce is impacted. \n\nNo other functionality on Spruce has been identified to be impacted at this point.",
"incident_id": "rd16m3h7hrv5",
"created_at": "2023-08-17T17:39:04.351-07:00",
"updated_at": "2023-08-17T17:39:44.072-07:00",
"display_at": "2023-08-17T17:39:04.000-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "qs34zyh8jwkq",
"name": "Call connectivity issues with Verizon",
"status": "resolved",
"created_at": "2023-07-26T06:57:13.290-07:00",
"updated_at": "2023-07-26T19:37:17.153-07:00",
"monitoring_at": null,
"resolved_at": "2023-07-26T08:00:43.000-07:00",
"impact": "major",
"shortlink": "https://stspg.io/r93mblc87gnd",
"started_at": "2023-07-26T06:57:13.284-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "8wk840cpwp7y",
"status": "resolved",
"body": "The issue was resolved around 7:45am PT on July 26 as per a statement from Verizon.",
"incident_id": "qs34zyh8jwkq",
"created_at": "2023-07-26T19:36:43.174-07:00",
"updated_at": "2023-07-26T19:37:17.142-07:00",
"display_at": "2023-07-26T08:00:43.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "partial_outage",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "5h3n5rv0q5ls",
"status": "identified",
"body": "Some users may be experiencing call connectivity issues due to an outage with Verison wireless network operator. Users may get a \"all circuits are busy\" message when placing a call. All other parts of the Spruce system are functioning normally. We are closely monitoring the situation and will post updates here.",
"incident_id": "qs34zyh8jwkq",
"created_at": "2023-07-26T06:57:13.365-07:00",
"updated_at": "2023-07-26T06:57:13.365-07:00",
"display_at": "2023-07-26T06:57:13.365-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "partial_outage"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "66fcwjfqjnty",
"name": "Delayed push notifications",
"status": "resolved",
"created_at": "2023-06-13T12:51:32.254-07:00",
"updated_at": "2023-06-13T15:56:01.442-07:00",
"monitoring_at": "2023-06-13T13:49:15.139-07:00",
"resolved_at": "2023-06-13T15:56:01.430-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/00hyvczgw189",
"started_at": "2023-06-13T12:51:32.249-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "qnqmvtckvlby",
"status": "resolved",
"body": "This incident has been resolved.",
"incident_id": "66fcwjfqjnty",
"created_at": "2023-06-13T15:56:01.430-07:00",
"updated_at": "2023-06-13T15:56:01.430-07:00",
"display_at": "2023-06-13T15:56:01.430-07:00",
"affected_components": [
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "zyhqp756qhds",
"status": "monitoring",
"body": "AWS has marked their incident as resolved, and all seems to be operating normally on our end.",
"incident_id": "66fcwjfqjnty",
"created_at": "2023-06-13T15:55:41.024-07:00",
"updated_at": "2023-06-13T15:55:41.024-07:00",
"display_at": "2023-06-13T15:55:41.024-07:00",
"affected_components": [
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "sb6m36554gtl",
"status": "monitoring",
"body": "Push notifications have all caught up. We will keep this ticket open until we have confirmation from AWS that the issue is resolved. We are no longer seeing any delays in processing push notifications.",
"incident_id": "66fcwjfqjnty",
"created_at": "2023-06-13T13:49:15.139-07:00",
"updated_at": "2023-06-13T13:49:15.139-07:00",
"display_at": "2023-06-13T13:49:15.139-07:00",
"affected_components": [
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "389nw5rcrptj",
"status": "investigating",
"body": "Push notifications are delayed due to an ongoing issue with our underlying infrastructure provider, Amazon Web Services (AWS). We have a workaround to resolve the delayed push notifications despite the ongoing AWS issue, and are working to deploy this workaround.\n\nMobile apps may receive push notifications for inbound and outbound messages in a delayed manner. The web application may refresh the state of the inbox in a delayed manner.",
"incident_id": "66fcwjfqjnty",
"created_at": "2023-06-13T12:51:32.326-07:00",
"updated_at": "2023-06-13T12:51:32.326-07:00",
"display_at": "2023-06-13T12:51:32.326-07:00",
"affected_components": [
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nftn1q0cs42j",
"name": "Spruce app notifications",
"status": "operational",
"created_at": "2021-12-10T13:25:35.080-08:00",
"updated_at": "2025-10-20T14:35:37.106-07:00",
"position": 8,
"description": "Real time notifications for new messages and video calls.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "3lcn7mkj1g8n",
"name": "Outbound SMS, email and fax delayed",
"status": "resolved",
"created_at": "2023-05-22T10:43:25.692-07:00",
"updated_at": "2023-05-22T11:45:54.585-07:00",
"monitoring_at": null,
"resolved_at": "2023-05-22T11:45:54.575-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/0142c7kmsgyf",
"started_at": "2023-05-22T10:43:25.687-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "6jw449dghfzh",
"status": "resolved",
"body": "All delayed outbound SMS, email and fax are now caught up and we are no longer experiencing a delay in sending messages. We will continue to monitor the system to ensure that the system is functioning smoothly.",
"incident_id": "3lcn7mkj1g8n",
"created_at": "2023-05-22T11:45:54.575-07:00",
"updated_at": "2023-05-22T11:45:54.575-07:00",
"display_at": "2023-05-22T11:45:54.575-07:00",
"affected_components": [
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "ly99sbskh7fv",
"status": "identified",
"body": "We continue to experience delayed SMS, email and fax. We have identified the problem to be with our telephony infrastructure provider and are actively working with them to resolve the issue.",
"incident_id": "3lcn7mkj1g8n",
"created_at": "2023-05-22T11:15:22.581-07:00",
"updated_at": "2023-05-22T11:15:22.581-07:00",
"display_at": "2023-05-22T11:15:22.581-07:00",
"affected_components": [
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "j0fh0sqbyjfh",
"status": "investigating",
"body": "We're currently seeing delays in outbound SMS, email and fax. \n\nNo impacts to inbound SMS, phone calls, video calls or secure messaging.",
"incident_id": "3lcn7mkj1g8n",
"created_at": "2023-05-22T10:43:25.791-07:00",
"updated_at": "2023-05-22T10:43:25.791-07:00",
"display_at": "2023-05-22T10:43:25.791-07:00",
"affected_components": [
{
"code": "6wxpvnqpkbsg",
"name": "Fax",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "6wxpvnqpkbsg",
"name": "Fax",
"status": "operational",
"created_at": "2018-08-14T18:45:20.310-07:00",
"updated_at": "2025-10-28T11:58:43.083-07:00",
"position": 5,
"description": "Inbound/Outbound Digital Fax",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "qwsh21hhc6l2",
"name": "Call connectivity issues",
"status": "resolved",
"created_at": "2023-05-18T17:00:29.498-07:00",
"updated_at": "2023-05-18T18:27:18.730-07:00",
"monitoring_at": null,
"resolved_at": "2023-05-18T18:27:18.716-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/gvfnttm8th90",
"started_at": "2023-05-18T17:00:29.493-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "sgfyqddd97f6",
"status": "resolved",
"body": "Our telecommunications infrastructure provider is seeing recovery on their end. Outbound and inbound calls should be fully functional at this point.",
"incident_id": "qwsh21hhc6l2",
"created_at": "2023-05-18T18:27:18.716-07:00",
"updated_at": "2023-05-18T18:27:18.716-07:00",
"display_at": "2023-05-18T18:27:18.716-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "51xk105z64x4",
"status": "investigating",
"body": "Some providers may be experiencing call connectivity issues due to failures on our underlying telecommunications provider's end. We are closely monitoring the situation and will post updates here.",
"incident_id": "qwsh21hhc6l2",
"created_at": "2023-05-18T17:00:29.561-07:00",
"updated_at": "2023-05-18T17:00:29.561-07:00",
"display_at": "2023-05-18T17:00:29.561-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "4kj9xjpmsy52",
"name": "Obi1032 and Obi1062 devices cannot make outbound calls",
"status": "resolved",
"created_at": "2023-05-17T08:33:29.884-07:00",
"updated_at": "2023-05-17T12:00:29.621-07:00",
"monitoring_at": "2023-05-17T09:25:42.227-07:00",
"resolved_at": "2023-05-17T12:00:29.603-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/3f02s06l437f",
"started_at": "2023-05-17T08:33:29.880-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "y1xfm7qmtnzb",
"status": "resolved",
"body": "This incident has been resolved.",
"incident_id": "4kj9xjpmsy52",
"created_at": "2023-05-17T12:00:29.603-07:00",
"updated_at": "2023-05-17T12:00:29.603-07:00",
"display_at": "2023-05-17T12:00:29.603-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "hxs496hm2j9h",
"status": "monitoring",
"body": "We have deployed a fix so that outbound calls from Obi1032 and Obi1062 devices now work. We are continuing to monitor the system to ensure there are no problems with outbound and inbound calls across all device types.",
"incident_id": "4kj9xjpmsy52",
"created_at": "2023-05-17T09:25:42.227-07:00",
"updated_at": "2023-05-17T09:25:42.227-07:00",
"display_at": "2023-05-17T09:25:42.227-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "drcmmlfcpz7p",
"status": "identified",
"body": "We have identified an issue where any medical practice using the Obi1032 and Obi1062 desk phones are unable to place outbound calls. They experience a busy dial tone when placing an inbound call. Inbound calls work fine.\n\nInbound and outbound calls from the smartphone application and other desk phone models continue to operate normally. No other aspect of the system is impacted.",
"incident_id": "4kj9xjpmsy52",
"created_at": "2023-05-17T08:33:29.937-07:00",
"updated_at": "2023-05-17T08:33:29.937-07:00",
"display_at": "2023-05-17T08:33:29.937-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "hkmkqbphqlxj",
"name": "Search \u0026 Contact Filters experiencing intermittent errors and slowness",
"status": "resolved",
"created_at": "2023-05-01T12:12:51.408-07:00",
"updated_at": "2023-05-01T15:28:45.623-07:00",
"monitoring_at": "2023-05-01T15:00:36.151-07:00",
"resolved_at": "2023-05-01T15:28:45.608-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/s58126scdtp2",
"started_at": "2023-05-01T12:12:51.403-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "y39zt23tyc01",
"status": "resolved",
"body": "System is fully operational now, with all search results having caught up. We're really sorry for the inconvenience caused here and have decided to operate with extra capacity to ensure that the system continues to operate normally moving forward.",
"incident_id": "hkmkqbphqlxj",
"created_at": "2023-05-01T15:28:45.608-07:00",
"updated_at": "2023-05-01T15:28:45.608-07:00",
"display_at": "2023-05-01T15:28:45.608-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "2frgs86jm1cz",
"status": "monitoring",
"body": "Search results and contacts are now loading without any errors. We are working to get search results to reflect latest results. Users should no longer have any issues creating conversations, sending bulk messages or opening contact lists.\n\nPhone calls, video calls, exchanging messages (fax, secure, email or SMS) are NOT impacted and operating normally.\n\nWill post an update as soon as we have search results all caught up.",
"incident_id": "hkmkqbphqlxj",
"created_at": "2023-05-01T15:00:36.151-07:00",
"updated_at": "2023-05-01T15:00:36.151-07:00",
"display_at": "2023-05-01T15:00:36.151-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "34744bksxps7",
"status": "identified",
"body": "We continue to actively work on the issue. We will post another update in another hour here as we assess progress of the fix.",
"incident_id": "hkmkqbphqlxj",
"created_at": "2023-05-01T13:52:09.520-07:00",
"updated_at": "2023-05-01T13:52:09.520-07:00",
"display_at": "2023-05-01T13:52:09.520-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "086g61zy6pw7",
"status": "identified",
"body": "We have identified the reason for the degraded performance and are actively working on resolving the issue.",
"incident_id": "hkmkqbphqlxj",
"created_at": "2023-05-01T12:32:53.744-07:00",
"updated_at": "2023-05-01T12:32:53.744-07:00",
"display_at": "2023-05-01T12:32:53.744-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "0nq64zbb10fv",
"status": "investigating",
"body": "Users may be experiencing degraded performance for the following actions:\n- Searching for contacts or conversations: may be giving errors or returning results in a delayed manner or returning stale results\n- Creating conversations: maybe return errors or experiencing a lag when searching for contacts\n- Sending bulk messages: may be experiencing a lag where stuck in Processing state for a while or just taking longer to complete\n- Opening contact lists: may be returning errors or experiencing a lag in returning results\n\nPhone calls, video calls, exchanging messages (fax, secure, email or SMS) is NOT impacted and operating normally.\n\nWe have identified the reason for the degraded performance and are actively working on resolving the issue.",
"incident_id": "hkmkqbphqlxj",
"created_at": "2023-05-01T12:12:51.473-07:00",
"updated_at": "2023-05-01T12:12:51.473-07:00",
"display_at": "2023-05-01T12:12:51.473-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "2vhkllyqpc37",
"name": "Search \u0026 Contact Filters experiencing intermittent errors and slowness",
"status": "resolved",
"created_at": "2023-04-18T13:19:37.940-07:00",
"updated_at": "2023-04-19T05:07:56.065-07:00",
"monitoring_at": "2023-04-18T13:57:55.133-07:00",
"resolved_at": "2023-04-18T15:38:58.477-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/gbt6fmw9xc2k",
"started_at": "2023-04-18T13:19:37.935-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "n55fzmv06406",
"status": "resolved",
"body": "This incident has been resolved.",
"incident_id": "2vhkllyqpc37",
"created_at": "2023-04-18T15:38:58.477-07:00",
"updated_at": "2023-04-18T15:38:58.477-07:00",
"display_at": "2023-04-18T15:38:58.477-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "1z166g76vj3x",
"status": "monitoring",
"body": "Spruce is back to being fully operational. We are monitoring the system to ensure that all continues to look good.",
"incident_id": "2vhkllyqpc37",
"created_at": "2023-04-18T13:57:55.133-07:00",
"updated_at": "2023-04-18T13:57:55.133-07:00",
"display_at": "2023-04-18T13:57:55.133-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "f6ym9xklg9df",
"status": "investigating",
"body": "We are continuing to investigate this issue.",
"incident_id": "2vhkllyqpc37",
"created_at": "2023-04-18T13:19:52.896-07:00",
"updated_at": "2023-04-18T13:19:52.896-07:00",
"display_at": "2023-04-18T13:19:52.896-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "d5z2wsbd4w5s",
"status": "investigating",
"body": "Users may be experiencing degraded performance for the following actions:\n- Searching for contacts or conversations: may be giving errors or returning results in a delayed manner or returning stale results\n- Creating conversations: maybe return errors or experiencing a lag when searching for contacts\n- Sending bulk messages: may be experiencing a lag where stuck in Processing state for a while or just taking longer to complete\n- Opening contact lists: may be returning errors or experiencing a lag in returning results\n\nPhone calls, video calls, exchanging messages (fax, secure, email or SMS) is NOT impacted and operating normally.\n\nWe have identified the reason for the degraded performance and are actively working on resolving the issue.",
"incident_id": "2vhkllyqpc37",
"created_at": "2023-04-18T13:19:38.023-07:00",
"updated_at": "2023-04-18T13:19:38.023-07:00",
"display_at": "2023-04-18T13:19:38.023-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "738nzt39cz7j",
"name": "Search related activities experiencing intermittent errors and slowness",
"status": "resolved",
"created_at": "2023-03-20T13:22:49.864-07:00",
"updated_at": "2023-03-20T16:08:27.538-07:00",
"monitoring_at": "2023-03-20T14:11:11.491-07:00",
"resolved_at": "2023-03-20T16:08:27.522-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/04sbwq0ck9xc",
"started_at": "2023-03-20T13:22:49.857-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "slw741gyybpd",
"status": "resolved",
"body": "System is fully operational. \n\nSummary: From 12:52 PT to 13:57 PT on March 20 2023, the following actions experienced degraded performance:\n- Searching for contacts or conversations: may be giving errors or returning results in a delayed manner or returning stale results\n- Creating conversations: maybe return errors or experiencing a lag when searching for contacts\n- Sending bulk messages: may be experiencing a lag where stuck in Processing state for a while or just taking longer to complete\n- Opening contact lists: may be returning errors or experiencing a lag in returning results",
"incident_id": "738nzt39cz7j",
"created_at": "2023-03-20T16:08:27.522-07:00",
"updated_at": "2023-03-20T16:08:27.522-07:00",
"display_at": "2023-03-20T16:08:27.522-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "25x8jcxnfvtn",
"status": "monitoring",
"body": "System is mostly recovered and we will continue to monitor to ensure that we do not see degraded performance for any actions.\n\nThere are still some stale search results being returned for contact and conversation search, but other than that, the system is fully operational at the moment.",
"incident_id": "738nzt39cz7j",
"created_at": "2023-03-20T14:11:11.491-07:00",
"updated_at": "2023-03-20T14:11:11.491-07:00",
"display_at": "2023-03-20T14:11:11.491-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "x86gqk6rnlj1",
"status": "identified",
"body": "We are continuing to work on a fix for this issue.",
"incident_id": "738nzt39cz7j",
"created_at": "2023-03-20T13:22:59.741-07:00",
"updated_at": "2023-03-20T13:22:59.741-07:00",
"display_at": "2023-03-20T13:22:59.741-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "23cgzv96p0mn",
"status": "identified",
"body": "Users may be experiencing degraded performance for the following actions:\n- Searching for contacts or conversations: may be giving errors or returning results in a delayed manner or returning stale results\n- Creating conversations: maybe return errors or experiencing a lag when searching for contacts\n- Sending bulk messages: may be experiencing a lag where stuck in Processing state for a while or just taking longer to complete\n- Opening contact lists: may be returning errors or experiencing a lag in returning results\n\nPhone calls, video calls, exchanging messages (fax, secure, email or SMS) is NOT impacted and operating normally.\n\nWe have identified the reason for the degraded performance and are actively working on resolving the issue.",
"incident_id": "738nzt39cz7j",
"created_at": "2023-03-20T13:22:49.922-07:00",
"updated_at": "2023-03-20T13:22:49.922-07:00",
"display_at": "2023-03-20T13:22:49.922-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "pgv7m3kwx08b",
"name": "Spruce experiencing issues with contact and conversation search related activities.",
"status": "postmortem",
"created_at": "2023-03-15T10:15:24.000-07:00",
"updated_at": "2023-04-14T13:21:34.265-07:00",
"monitoring_at": null,
"resolved_at": "2023-03-16T15:23:45.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/4zhkp3sy8s9r",
"started_at": "2023-03-15T10:15:24.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "9qtp223966x4",
"status": "postmortem",
"body": "## **Summary \\+ customer impact**\n\nThe Spruce system experienced degraded performance from 10:15am PT to 6:50pm PT on Match 15 2023. During this time:\n\n* Searching for contacts or conversations resulted in errors, slowness or stale results\n* Searching contacts to create conversations resulted in errors or slowness\n* Bulk messages sent were delayed due to being stuck in Processing state for a while or just taking longer to complete\n* Opening contact lists likely returned errors or experiencing slowness returning results\n* Contacts exports were delayed due to being stuck in Processing state for a while or just taking longer to complete\n\nThe degraded performance was caused due to inefficient data distribution across the search cluster where one of our data nodes experienced heavy load and was unable to process any new indexing and search operations.\n\n## **Analysis**\n\nThe Spruce engineering team immediately reacted to the monitoring alarms that were triggered to investigate the issue. The issue was caused due to one of the data nodes storing significantly larger amounts of data, compared to the other data nodes in the cluster, which resulted in the node taking longer to process requests. The number of requests piled up over time putting the node under heavier load, and getting to a point where the request queue was exhausted, resulting in new requests being rejected.\n\nThe engineering team reviewed and analyzed the cluster configuration and performance metrics in detail, and made the following steps to resolve the issue:\n\n* Adjusted the data distribution strategy so the data can be equally allocated across the data nodes in the search cluster. The old strategy was inefficient because the size of the stored data grew significantly over time and it could not support the demands of indexing and search operations.\n* Increased the number of data nodes and reallocated the data equally across the nodes. This was done in the background with minimal impact on the searching and indexing of new data. With the new configuration, data stored was uniformly distributed across all nodes.\n\nDuring the degraded performance, the other data nodes in the cluster were fully operational and any requests that were routed to them were processed successfully. Also, all the indexing requests were successfully queued and processed after the issue was resolved.\n\n## **Action items**\n\n* Create additional monitoring alerts for search and indexing operations latency, so that any potential issues can be early detected.\n* Review the cluster configuration and performance metrics in detail every 3-6 months.\n* Improve the clean-up strategy for unused data to reduce space usage.\n* Create an internal strategy with clearly defined steps that can be taken in order to quickly troubleshoot and resolve issues like this one and thus minimize the impact on the clients.\n* Increase general knowledge about the search cluster and its configuration within the engineering team.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-04-14T13:21:27.172-07:00",
"updated_at": "2023-04-14T13:21:34.262-07:00",
"display_at": null,
"affected_components": null,
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "b0l6501tnjzk",
"status": "resolved",
"body": "The system is fully operational as per our active monitoring over the last 10 hours.\n\nSummary: \nFrom 10:15am PT March 15 to 6:50pm PT March 15, the following actions on Spruce experienced degraded performance:\n- Searching for conversations and contacts either took long or failed\n- Contact filters frequently failed to load when clicked into \n- When starting a new conversation, contact suggestions either took long or failed to load, making it challenging to start new conversations\n- Bulk actions (messaging, tagging, deleting) took longer than expected to complete, but eventually completed\n- Newly created contacts, conversations and messages during this time period were not searchable. The new items eventually became searchable 6:50pm PT onwards\n- Successful searches for contacts and conversations may have brought up stale results, where an update to a contact or conversation was not reflected in search results. The updated items were eventually updated in the search to reflect their latest versions\n\nThere was no impact to calls, SMS, Fax, Secure Messaging, Email or Fax during this time. \n\nWe will post a postmortem to the incident soon.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-16T15:23:45.062-07:00",
"updated_at": "2023-03-16T15:28:22.765-07:00",
"display_at": "2023-03-16T15:23:45.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "dwy4q4q5gljk",
"status": "identified",
"body": "The redistribution of data in the cluster is still in progress (note that this happens in the background with minimal impact to searching and indexing of new data). We have been closely monitoring the situation throughout the night. We also increased the capacity of the cluster to accommodate for the redistribution of data and to insure that we are in better shape for today. We have ~20% of redistribution remaining that we believe will have a long standing improvement to the overall performance.\n\nThe metrics so far are looking healthy with no signs pointing to poor performance or increased error rate. \n\nWe will report back here once the redistribution completes or if we see any signs pointing to degraded performance.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-16T07:58:27.990-07:00",
"updated_at": "2023-03-16T07:58:27.990-07:00",
"display_at": "2023-03-16T07:58:27.990-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "5nf6xtzvs1b6",
"status": "identified",
"body": "Indexing of data has now caught up such that successfully searches for any contacts, conversations and messages will bring up up to date results. We are continuing to work on better distributing the data across the cluster.\n\nWe are not experiencing poor performance or intermittent errors currently. This is likely due to the decreased overall traffic in the system given time of day. That being said, we continue to work on reducing the likelihood of this problem continuing into business day tomorrow.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-15T19:11:57.897-07:00",
"updated_at": "2023-03-15T19:11:57.897-07:00",
"display_at": "2023-03-15T19:11:57.897-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "47nmwqrqq4w7",
"status": "identified",
"body": "We have identified a potential cause for the intermittent failures with the search cluster. We are going to work towards better distributing the data across the cluster so as to increase overall performance and reduce the error rate.\n\nTo recap, due to the errors throughout the day:\n- Searching for conversations, messages or contacts may have failed\n- Bulk messages may have taken longer to complete than usual\n- Newly created contacts, conversations and messages may not have shown up when searching\n- Updates to contacts may not have been searchable \n\nWe will continue to work through the evening to reindex the data so as to better distribute it across the cluster and keep this incident up to date as we make progress here.\n\nWe're really sorry for the inconvenience this is causing to your workflows.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-15T18:16:18.959-07:00",
"updated_at": "2023-03-15T18:16:18.959-07:00",
"display_at": "2023-03-15T18:16:18.959-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "zx8gnq99bn4v",
"status": "identified",
"body": "We continue to investigate this issue. Note that some bulk message operations may take long to complete or may get stuck in a particular state given that the bulk message operations also face similar errors when querying contact lists.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-15T15:31:33.019-07:00",
"updated_at": "2023-03-15T15:31:33.019-07:00",
"display_at": "2023-03-15T15:31:33.019-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "h6xsn542b5s4",
"status": "identified",
"body": "We continue to work on the issue here to reduce the intermittent errors while searching for contacts or loading contact lists. Note that there is no impact to phone calls, SMS routing, loading of inbox, exchanging secure messages, or video calling. Bulk messages will continue to send during this time, albeit in a delayed fashion given that bulk messages work off of contact lists.\n\nWe will update here as we make progress against the performance issue here.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-15T13:29:01.394-07:00",
"updated_at": "2023-03-15T13:29:01.394-07:00",
"display_at": "2023-03-15T13:29:01.394-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "73qpk9n4w691",
"status": "identified",
"body": "We are continuing to work on a fix for this issue.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-15T12:31:55.716-07:00",
"updated_at": "2023-03-15T12:31:55.716-07:00",
"display_at": "2023-03-15T12:31:55.716-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "fv0pfclvk6d3",
"status": "identified",
"body": "We are continuing to work on a fix for this issue.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-15T12:27:28.599-07:00",
"updated_at": "2023-03-15T12:27:28.599-07:00",
"display_at": "2023-03-15T12:27:28.599-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "t8g7rnvl5c84",
"status": "identified",
"body": "We are investigating an issue with contact and conversation search related activities.",
"incident_id": "pgv7m3kwx08b",
"created_at": "2023-03-15T12:26:24.613-07:00",
"updated_at": "2023-03-15T12:31:50.191-07:00",
"display_at": "2023-03-15T10:15:24.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "hwvswnprzz23",
"name": "Spruce Inbox failing to load",
"status": "postmortem",
"created_at": "2023-01-25T07:17:26.780-08:00",
"updated_at": "2023-01-26T14:02:32.663-08:00",
"monitoring_at": "2023-01-25T10:05:16.471-08:00",
"resolved_at": "2023-01-25T17:44:13.000-08:00",
"impact": "major",
"shortlink": "https://stspg.io/w39k9jnpc69r",
"started_at": "2023-01-25T07:17:26.773-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "hy7jbbvdf3g9",
"status": "postmortem",
"body": "## **Summary \\+ customer impact**\n\nThe Spruce system experienced an outage from 6:45am PT to 9:45am PT on January 25 2023. During this time:\n\n* The Spruce Inbox failed to load for patients and providers\n* Providers could not place voice or video calls to patients given that the inbox would not load\n* Providers could not send SMS or secure messages to patients\n* Patients could not send secure messages to practices\n* Inbound fax, SMS and email arrived to the Spruce inbox in a delayed fashion.\n* Inbound calls were operational during this time, however voicemails arrived to the Spruce inbox in a delayed fashion.\n* Workflow automation was executed albeit in a delayed fashion.\n\nThe outage was caused due to CPU exhaustion on one of the core databases. The engineering team believes the CPU exhaustion to be caused due to a frequently run inefficient query that was optimized as part of the fix. The engineering team will be proactively and closely monitoring the system over the rest of the week to ensure that there is no signs of database CPU exhaustion during peak hours of the day.\n\n## **Analysis**\n\nThe Spruce engineering team immediately reacted to the monitoring alarms that were triggered to investigate the issue. The issue is believed to be caused due to an inefficient query that is frequently executed across the entire customer base that built up over time and finally tilted one of the database into CPU exhaustion. The engineering team deployed a fix for the inefficient query at around 9am PT.\n\nIt took 45 minutes to bring the system back up once the query optimization fix was deployed because the engineering team wanted to ensure that bringing the system back all at once did not cause resource exhaustion on various parts of the system. So they brought various components up in a serialized manner while constantly monitoring the CPU utilization on the impacted database.\n\nThe backup communications system was activated around 8am PT so that any customer had registered their contact information received notifications for inbound calls and SMS over email via a secure expiring URL.\n\n## **Action items**\n\n* Make the Backup communications system self-service in the product so that anyone can register for it.\n* Automatically send an SMS in response to an inbound SMS when the backup system is activated. Currently, we only send an automated text if the provider has signed up for notifications from the backup system, rather for any inbound SMS.\n* Install a global rate limiter per account to ensure at the API layer so that we have protections around client applications constantly retrying and causing a large spike in traffic when the platform is experiencing issues.\n* Gain deeper insight into our asynchronous workers through metrics so that we can look at the overall health of the workers running across the platform and ensure there are no runaway workers causing platform wide issues.\n* Make it part of the engineering on-call person’s responsibilities to proactively monitor database performance metrics and identify any database statements that need optimizing before they build up over time\n* Clean up ever-growing tables to ensure that they are not impacting general database performance across key services\n* Investigate how AWS Performance Insights API can be leveraged to automatically notify the engineering team of database queries that take too long to execute or scan too many rows.",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T17:45:57.096-08:00",
"updated_at": "2023-01-26T14:02:32.660-08:00",
"display_at": null,
"affected_components": null,
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "z3b8gl92q636",
"status": "resolved",
"body": "The Spruce system experienced an outage from 6:45am PT to 9:45am PT on January 25, 2023. During this time:\n\n- The Spruce Inbox failed to load for patients and providers\n- Providers could not place voice or video calls to patients, given that the inbox would not load\n- Providers could not send SMS or secure messages to patients\n- Patients could not send secure messages to practices\n- Inbound fax, SMS messages, and email arrived to the Spruce inbox in a delayed fashion\n- Inbound calls were operational during this time; however, voicemails arrived to the Spruce inbox in a delayed fashion\n- Workflow automation was executed, albeit in a delayed fashion\n\nThe outage was caused by CPU exhaustion on one of the core databases. The engineering team believes the CPU exhaustion to have been the result of a frequently run inefficient query that built up over time, and which was optimized as part of the fix for this incident. The engineering team will be proactively and closely monitoring the system in the next days to ensure that there is no sign of database CPU exhaustion, including especially during peak hours of the day.\n\nWe know how important it is for Spruce to be fully operational at all times. Working to build and maintain a medical communications system gives us all immense energy on a daily basis and is not a job we take lightly. We're very sorry for the outage and the impact to practices and patients. We will continue to work hard in pursuit of a highly available and reliable service. If you have any questions at all, please don't hesitate to reach us at support@sprucehealth.com.",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T17:44:13.195-08:00",
"updated_at": "2023-01-25T19:16:37.850-08:00",
"display_at": "2023-01-25T17:44:13.000-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "fmtz5swgn466",
"status": "monitoring",
"body": "The system should be fully functional at this point. We are continuing to monitor the system. We will post an incident report once we've had a chance to investigate more deeply here.",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T10:05:16.471-08:00",
"updated_at": "2023-01-25T10:05:16.471-08:00",
"display_at": "2023-01-25T10:05:16.471-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "thb8kv9prv9z",
"status": "identified",
"body": "We are starting to see some recovery and are slowly ramping the system back up to fully serviceable to see the impact on database and CPU in general. We'll keep updating this page as we have more to share.",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T09:45:40.711-08:00",
"updated_at": "2023-01-25T09:45:40.711-08:00",
"display_at": "2023-01-25T09:45:40.711-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "major_outage",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "major_outage",
"new_status": "degraded_performance"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "major_outage",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "kp8d8d7nggpf",
"status": "investigating",
"body": "We have made a database optimization for a high frequency lookup. We have intentionally brought down the API layer that clients connect to while we work to ensure that the rest of the system is functional. Once all asynchronous work has been completed, we will turn on the API layer slowly to ensure that we are not seeing any CPU performance issues again.",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T09:21:18.367-08:00",
"updated_at": "2023-01-25T09:21:18.367-08:00",
"display_at": "2023-01-25T09:21:18.367-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "major_outage",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "bf68qkxdgglx",
"status": "investigating",
"body": "We continue to investigate the issue with no root cause yet unfortunately. We are all hands on deck working to identify the reason for the outage.",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T08:52:20.621-08:00",
"updated_at": "2023-01-25T08:52:20.621-08:00",
"display_at": "2023-01-25T08:52:20.621-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "major_outage",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "gt71r5xxxngg",
"status": "investigating",
"body": "The backup system for notifying providers of incoming SMS, Fax, call events and voicemails has now been activated. Anyone that has registered contact information for our backup system will now get notified over email. You can read more about the backup system here: https://help.sprucehealth.com/article/424-spruce-backup-system",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T08:20:09.341-08:00",
"updated_at": "2023-01-25T08:20:09.341-08:00",
"display_at": "2023-01-25T08:20:09.341-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "major_outage",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "y132xzf76v1t",
"status": "investigating",
"body": "We have not identified the root cause yet. The inbox continues to be down for most. We are actively investigating this issue.",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T07:51:00.511-08:00",
"updated_at": "2023-01-25T07:51:00.511-08:00",
"display_at": "2023-01-25T07:51:00.511-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "major_outage",
"new_status": "major_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "major_outage",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "g2xqk50mml86",
"status": "investigating",
"body": "The Spruce inbox is unable to load at the moment and consequently patients and providers are unable to view/send messages, fax or make calls. Inbound calls should be working. Inbound fax likely delayed.",
"incident_id": "hwvswnprzz23",
"created_at": "2023-01-25T07:17:26.937-08:00",
"updated_at": "2023-01-25T07:17:26.937-08:00",
"display_at": "2023-01-25T07:17:26.937-08:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "major_outage"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "major_outage"
},
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "nj4tg0xb6sp3",
"name": "Spruce Inbox intermittently failed to load",
"status": "resolved",
"created_at": "2022-12-19T05:08:00.000-08:00",
"updated_at": "2022-12-27T10:23:52.007-08:00",
"monitoring_at": null,
"resolved_at": "2022-12-19T05:08:00.000-08:00",
"impact": "minor",
"shortlink": "https://stspg.io/22nlbwpmrs53",
"started_at": "2022-12-27T10:23:38.920-08:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "6684fdqbz4rk",
"status": "resolved",
"body": "Between 5:08am and 5:24am Pacific Time on December 19th, the Spruce Inbox failed to load for some users. Users experienced errors opening the inbox, or a particular conversation. Users may have also experienced intermittent failures when placing an outbound call or posting a message or sending a fax. \n\nThere was no impact to inbound calls or video calls at this time. Some inbound SMS, fax and call events arrived in the inbox in a delayed manner during this time.\n\nFrom our investigation, the intermittent failures were due to one of the services experiencing a malfunctioning task. We plan to look into this further for how to prevent a particular malfunctioning task from causing intermittent platform wide failures.",
"incident_id": "nj4tg0xb6sp3",
"created_at": "2022-12-27T10:23:38.966-08:00",
"updated_at": "2022-12-27T10:23:38.966-08:00",
"display_at": "2022-12-19T05:08:00.000-08:00",
"affected_components": null,
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [],
"reminder_intervals": null
},
{
"id": "041762s1cmy6",
"name": "Spruce experiencing issues with outbound calls from the android mobile app",
"status": "postmortem",
"created_at": "2022-11-03T09:04:08.081-07:00",
"updated_at": "2022-11-03T19:52:26.621-07:00",
"monitoring_at": "2022-11-03T09:09:46.235-07:00",
"resolved_at": "2022-11-03T09:10:38.000-07:00",
"impact": "major",
"shortlink": "https://stspg.io/l0zzwprflnt9",
"started_at": "2022-11-03T09:04:08.076-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "8wp2rgrfrbp9",
"status": "postmortem",
"body": "## Summary\n\nAt 8:23am PT, we released software that resulted in outbound calls from the Android app failing to work. This was due to software that was incompatible with the latest version of the Android app. We identified the issue at 9:04am PT and quickly root caused it to the software rollout. We deployed a fix at 9:08am PT. Outbound calls on the Spruce Android app were non-functional for customers running the latest version of the app from 8:23am PT - 9:08am PT \\(45 minutes\\).\n\nOutbound calling from desk phones, iOS or web-apps were not impacted. No other aspect of the system was impacted.\n\nWe are internally discussing ways to reduce the likelihood of incompatible software rollouts like the one that happened today. We understand how important it is for providers to be able to place outbound calls from all platforms at all times, and we’re very sorry for the inconvenience and the business impact here.",
"incident_id": "041762s1cmy6",
"created_at": "2022-11-03T19:43:39.740-07:00",
"updated_at": "2022-11-03T19:51:37.022-07:00",
"display_at": null,
"affected_components": null,
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "lkxr7bbhhts1",
"status": "resolved",
"body": "This incident has been resolved.",
"incident_id": "041762s1cmy6",
"created_at": "2022-11-03T15:23:38.964-07:00",
"updated_at": "2022-11-03T19:52:26.607-07:00",
"display_at": "2022-11-03T09:10:38.000-07:00",
"affected_components": [
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "partial_outage",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "wy9cyj80kgcl",
"status": "monitoring",
"body": "A fix has been implemented and we are monitoring the results.",
"incident_id": "041762s1cmy6",
"created_at": "2022-11-03T09:09:46.235-07:00",
"updated_at": "2022-11-03T09:09:46.235-07:00",
"display_at": "2022-11-03T09:09:46.235-07:00",
"affected_components": [
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "partial_outage",
"new_status": "partial_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "y91v30178pqz",
"status": "identified",
"body": "The issue has been identified and a fix is being implemented.",
"incident_id": "041762s1cmy6",
"created_at": "2022-11-03T09:06:09.609-07:00",
"updated_at": "2022-11-03T09:06:09.609-07:00",
"display_at": "2022-11-03T09:06:09.609-07:00",
"affected_components": [
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "partial_outage",
"new_status": "partial_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "7brpmczjrx0d",
"status": "investigating",
"body": "We are investigating an issue with outbound calls from the android mobile app",
"incident_id": "041762s1cmy6",
"created_at": "2022-11-03T09:04:08.154-07:00",
"updated_at": "2022-11-03T09:04:08.154-07:00",
"display_at": "2022-11-03T09:04:08.154-07:00",
"affected_components": [
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "partial_outage"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "g1fnb6vv0316",
"name": "Contact lists and search results failing to load intermittently",
"status": "resolved",
"created_at": "2022-10-24T12:40:50.000-07:00",
"updated_at": "2022-10-26T01:16:07.444-07:00",
"monitoring_at": "2022-10-25T00:49:39.050-07:00",
"resolved_at": "2022-10-25T12:55:00.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/pfxkcmwjn6x3",
"started_at": "2022-10-24T12:40:50.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "r231c7tftlnc",
"status": "resolved",
"body": "As of 12:55pm PT on October 25 2022, we started to see recovery and found the system to return to being fully operational. There was a re-allocation of data across the nodes that store and process the search engine data. The re-allocation seems to have helped the overall recovery. \n\nWe will continue to investigate ways in which we can further optimize our search engine database to prevent such spikes in search latency.",
"incident_id": "g1fnb6vv0316",
"created_at": "2022-10-26T01:15:55.576-07:00",
"updated_at": "2022-10-26T01:16:07.426-07:00",
"display_at": "2022-10-25T12:55:00.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "9wsmzj00nn0y",
"status": "investigating",
"body": "Note that while we are experiencing this performance issue, new contacts, updates to contacts and messages may not be reflected in search results. This is because there is a backup in indexing of new data into the search engine.\n\nAlso note that failures may be intermittent. Contact lists may fail to load but then if you refresh the page it may end up loading. This will also be the experience with conversation and contact search results.",
"incident_id": "g1fnb6vv0316",
"created_at": "2022-10-25T12:17:14.592-07:00",
"updated_at": "2022-10-25T12:17:14.592-07:00",
"display_at": "2022-10-25T12:17:14.592-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "px2w578pbcsl",
"status": "investigating",
"body": "The performance issues seem to have started again as of 11:02am PT today, despite the infrastructure improvements. We are continuing to investigate to identify an performance bottlenecks.",
"incident_id": "g1fnb6vv0316",
"created_at": "2022-10-25T11:17:23.453-07:00",
"updated_at": "2022-10-25T11:17:23.453-07:00",
"display_at": "2022-10-25T11:17:23.453-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "jjflkm5j51m6",
"status": "monitoring",
"body": "We have made some underlying infrastructure changes that should help alleviate the performance issues that we were experiencing. We will continue to monitor the overall health of the database that backs searches and contact lists during normal business operations today (October 25 2022) to ensure that the issue is fully resolved and the platform is back to being functional.",
"incident_id": "g1fnb6vv0316",
"created_at": "2022-10-25T00:49:39.050-07:00",
"updated_at": "2022-10-25T00:49:39.050-07:00",
"display_at": "2022-10-25T00:49:39.050-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "587j9nhzn541",
"status": "investigating",
"body": "We are currently investigating an issue where contact lists and search results fail to load intermittently. \n\nThere should be no impact to the Spruce inbox, inbound/outbound calls, video calls or exchanging of messages over SMS, secure messaging, email and fax.",
"incident_id": "g1fnb6vv0316",
"created_at": "2022-10-24T13:49:50.070-07:00",
"updated_at": "2022-10-24T13:50:11.871-07:00",
"display_at": "2022-10-24T12:40:50.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "6621xkh7cxyj",
"name": "Delayed SMS delivery to AT\u0026T phone numbers",
"status": "resolved",
"created_at": "2022-10-19T14:08:56.000-07:00",
"updated_at": "2022-10-20T14:12:34.055-07:00",
"monitoring_at": "2022-10-19T14:08:56.000-07:00",
"resolved_at": "2022-10-19T19:27:23.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/xws9qfbjs2wt",
"started_at": "2022-10-19T14:08:56.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "v2fm7y5q2q0m",
"status": "resolved",
"body": "Our telecom partner updated us last evening around 7:27pm (October 19) that the issue was resolved on their end and they were no longer seeing delays in sending SMS to AT\u0026T.",
"incident_id": "6621xkh7cxyj",
"created_at": "2022-10-20T14:12:23.411-07:00",
"updated_at": "2022-10-20T14:12:34.042-07:00",
"display_at": "2022-10-19T19:27:23.000-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "cqqfmwl51k0b",
"status": "monitoring",
"body": "Our telecom partner just posted an update saying that they continue to experience messaging sending delays and they are working closely with their partner to resolve the issue. They will post an update soon, likely in the next 6-8 hours on this matter.",
"incident_id": "6621xkh7cxyj",
"created_at": "2022-10-19T17:10:51.114-07:00",
"updated_at": "2022-10-19T17:10:51.114-07:00",
"display_at": "2022-10-19T17:10:51.114-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "j6wp09cv5pf5",
"status": "monitoring",
"body": "Our underlying telecommunications infrastructure provider is experiencing SMS delivery delays when when sending messages to AT\u0026T phone numbers. The message may appear as sent in your Spruce application, but may take minutes to get to the recipient.\n\nThis incident with our underlying provider is also impacting verification code delivery over SMS. So if a patient or provider tries to create an account or log in, and their cell phone number is with AT\u0026T, they may not get the verification code for minutes. In this case, it is likely best for the user to try the \"Call me with code\" option accessible on all platforms.\n\nWe are actively monitoring this incident and will post an update as soon as we have one.",
"incident_id": "6621xkh7cxyj",
"created_at": "2022-10-19T14:08:56.635-07:00",
"updated_at": "2022-10-19T14:09:04.364-07:00",
"display_at": "2022-10-19T14:08:56.000-07:00",
"affected_components": [
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "k85lkzcxvq8p",
"name": "[RESOLVED] Delayed inbound sms, voicemails and call events",
"status": "postmortem",
"created_at": "2022-10-18T10:05:21.000-07:00",
"updated_at": "2022-10-18T17:31:42.651-07:00",
"monitoring_at": null,
"resolved_at": "2022-10-18T11:53:11.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/df2q6w5hnwxc",
"started_at": "2022-10-18T10:05:21.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "sjs6mrs41grz",
"status": "postmortem",
"body": "# Summary\n\nThe reason for the delayed messages was because of communication with our transcription provider timing out to transcribe voicemails. The timeout on uploading a recording to the provider was not correctly tuned, leading to a build-up of messages that needed to be processed by a set of application workers and causing a backlog of messages that needed to be processed. The messages were being processed albeit in a delayed manner due to the communication issues. \n\nSpruce was made aware of the issue via multiple customer complaints and the engineering team started investigating as soon as the issue was escalated.\n\n# **Action items to mitigate future impact**\n\n* Add an alarm on the application worker responsible for processing transcriptions and SMS. Note that we already had alarms in place for all but one of the workers. This will help ensure that should an issue like this arise again, we’ll be notified asap.\n* Fine tune the timeout in communication with the transcription provider to prevent a build-up in the event of communication errors in the future.",
"incident_id": "k85lkzcxvq8p",
"created_at": "2022-10-18T17:26:42.324-07:00",
"updated_at": "2022-10-18T17:31:42.648-07:00",
"display_at": null,
"affected_components": null,
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "xfqgcl9tbw71",
"status": "resolved",
"body": "The issue was resolved and the system returned to being fully functional at around 11:53 am PT.",
"incident_id": "k85lkzcxvq8p",
"created_at": "2022-10-18T17:26:11.667-07:00",
"updated_at": "2022-10-18T17:26:21.624-07:00",
"display_at": "2022-10-18T11:53:11.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "rt2jcbg32rxy",
"status": "identified",
"body": "From 10:05am PT to 11:53am PT on October 18 2022, voicemails, inbound sms and inbound call events reached provider's Spruce inboxes in a delayed manner. The events that were delayed had an indication in the message itself for how long they were delayed by.\n\nThere was no impact to inbound calls, outbound calls, secure message exchanges, video calls, email or fax. \n\nSpruce identified this issue in response to customer complaints rather than the proactive monitoring in place for the system in general.",
"incident_id": "k85lkzcxvq8p",
"created_at": "2022-10-18T17:25:21.115-07:00",
"updated_at": "2022-10-18T17:25:29.323-07:00",
"display_at": "2022-10-18T10:05:21.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "g1nsmbx5d5wr",
"name": "Spruce-Elation integration is erroring",
"status": "postmortem",
"created_at": "2022-10-12T08:42:10.377-07:00",
"updated_at": "2022-10-12T11:24:34.160-07:00",
"monitoring_at": null,
"resolved_at": "2022-10-12T11:16:13.985-07:00",
"impact": "critical",
"shortlink": "https://stspg.io/xw9m1tghqz78",
"started_at": "2022-10-12T08:42:10.372-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "tyy48vfnld0f",
"status": "postmortem",
"body": "**Summary**\n\nElation changed the error code being returned to indicate expired credentials. Spruce had to update the error code they were looking for to refresh credentials before making an API request. Spruce was notified of this issue due to a customer complaint, 10 hours after the issue started. Spruce identified that there was no code deployment on their end, so engaged Elation to help investigate. \n\n**Action items to mitigate the impact in the future**\n\n* Spruce to put in a fix to resolve the issue \\(done\\)\n* On Elation end \\(not in our control so cannot say if will be done or not, but Spruce has communicated with Elation on these\\):\n\n * Elation to consider reverting the error code change \n * Elation to consider improved error messages if credentials have expired\n \n* Spruce to have a quicker escalation path with Elation \\(Elation shared email addresses that Spruce should reach out to, in addition to Slack\\)\n* Spruce to investigate how they can get proactively notified of elevated integration related error",
"incident_id": "g1nsmbx5d5wr",
"created_at": "2022-10-12T11:16:46.482-07:00",
"updated_at": "2022-10-12T11:24:34.158-07:00",
"display_at": null,
"affected_components": null,
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "90frgz00kn2k",
"status": "resolved",
"body": "We deployed a fix at 11:08 am PT Oct 12 2022 and can confirm as of 11:15am PT that the Spruce-Elation integration is now fully functional again.",
"incident_id": "g1nsmbx5d5wr",
"created_at": "2022-10-12T11:16:13.985-07:00",
"updated_at": "2022-10-12T11:16:13.985-07:00",
"display_at": "2022-10-12T11:16:13.985-07:00",
"affected_components": [
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "major_outage",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "1s66q3ghzjl5",
"status": "identified",
"body": "We have identified the cause to be a deployment by the Elation development team last night (around 9:40pm PT Oct 11 2022) where there was a change in assumptions for when an API partner (such as Spruce) should re-authenticate credentials for an integration. Consequently, most integrations were failing given that Spruce was not refreshing credentials when it should.\n\nWe are making a change to adapt to the updated assumptions, while the Elation team is evaluating if they should revert back to previous assumptions. \n\nWe should have a fix rolled out in the next 30 minutes.",
"incident_id": "g1nsmbx5d5wr",
"created_at": "2022-10-12T09:58:01.672-07:00",
"updated_at": "2022-10-12T09:58:09.256-07:00",
"display_at": "2022-10-12T09:45:01.000-07:00",
"affected_components": [
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "major_outage",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "4shy0mxxzddm",
"status": "investigating",
"body": "We are currently experiencing an issue where the Spruce-Elation integration is erroring. Any patient a provider adds to Elation may not automatically show up in Spruce, if that's what you expect. Any demographic changes made to a patient in Spruce or Elation will not sync to Elation or Spruce. Any attempt to sync messages from Spruce to Elation will not work.\n\nWe are investigating this issue in partnership with Elation as we speak to root cause it.",
"incident_id": "g1nsmbx5d5wr",
"created_at": "2022-10-12T08:42:10.444-07:00",
"updated_at": "2022-10-12T08:42:10.444-07:00",
"display_at": "2022-10-12T08:42:10.444-07:00",
"affected_components": [
{
"code": "xdrmpzwhxky9",
"name": "Integrations",
"old_status": "operational",
"new_status": "major_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "xdrmpzwhxky9",
"name": "Integrations",
"status": "operational",
"created_at": "2022-10-12T08:39:52.774-07:00",
"updated_at": "2025-10-20T12:50:00.997-07:00",
"position": 9,
"description": "This component comprises all native integrations between Spruce and external systems, such as Spruce-Elation, Spruce-Hint and Spruce-Stripe integrations.",
"showcase": true,
"start_date": "2022-10-12",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "q0cnqn2zpsms",
"name": "Sent messages reappearing in the composer bar immediately after being sent",
"status": "resolved",
"created_at": "2022-10-08T01:05:00.000-07:00",
"updated_at": "2022-10-08T11:30:56.087-07:00",
"monitoring_at": null,
"resolved_at": "2022-10-08T01:05:00.000-07:00",
"impact": "none",
"shortlink": "https://stspg.io/74dz1mfbgdmj",
"started_at": "2022-10-08T11:30:56.038-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "wr7wk5xld9fy",
"status": "resolved",
"body": "From approximately 1:05 AM to 11:23 AM Pacific time, the Spruce web app would intermittently restore an already-sent message to the compose bar.",
"incident_id": "q0cnqn2zpsms",
"created_at": "2022-10-08T11:30:56.079-07:00",
"updated_at": "2022-10-08T11:30:56.079-07:00",
"display_at": "2022-10-08T01:05:00.000-07:00",
"affected_components": null,
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [],
"reminder_intervals": null
},
{
"id": "cxnkkhwnp440",
"name": "Patients' inbox and messages failing to load on web",
"status": "resolved",
"created_at": "2022-09-13T00:09:00.000-07:00",
"updated_at": "2022-09-14T09:04:22.471-07:00",
"monitoring_at": null,
"resolved_at": "2022-09-13T00:09:00.000-07:00",
"impact": "none",
"shortlink": "https://stspg.io/j1y6qbyb3svd",
"started_at": "2022-09-13T00:09:00.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "bpypfb5d5rty",
"status": "resolved",
"body": "During approximately 11 hours (00:09 to 11:07 PDT), patients were not able to view conversations or messages on the Spruce web app. This impacted most patients who used the web app (the exception being those loading the web app from a browser bookmark). The issue was caused by a server-side change in logic intended to increase performance, which caused unexpected conditions for the patient web app. The result was that patients were not able to see conversations during this period.",
"incident_id": "cxnkkhwnp440",
"created_at": "2022-09-14T09:00:16.065-07:00",
"updated_at": "2022-09-14T09:04:22.469-07:00",
"display_at": "2022-09-13T00:09:00.000-07:00",
"affected_components": null,
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [],
"reminder_intervals": null
},
{
"id": "2tvtp6sb23wf",
"name": "Delayed notifications",
"status": "postmortem",
"created_at": "2022-09-09T08:57:20.000-07:00",
"updated_at": "2022-09-09T12:29:51.506-07:00",
"monitoring_at": null,
"resolved_at": "2022-09-09T11:34:36.884-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/k3s6g6bgs0wx",
"started_at": "2022-09-09T08:57:20.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "cgpxs2jdcv71",
"status": "postmortem",
"body": "# Summary\n\nOur cloud infrastructure provider, AWS reported elevated errors and latency for the System Manager Parameter Store service in US-East-1. You can see their report [here](https://www.loom.com/i/ac62abd3a9c5448195304c2f2c65c25b). This AWS incident impacted the ability for AWS Lambda functions to execute since they could not read parameter values from the parameter store. This AWS incident in turn impacted the Spruce platform as described below.\n\n# Context\n\nThe Spruce platform leverages AWS Lambdas \\(server-less functions\\) to process inbound SMS, user-facing app-based push notifications, and badge count updates for smartphones and the web. The benefit of the Lambdas is that they automatically scale up in times of high demand and scale down to maintain a minimum number of serverless functions. \n\nThe Spruce platform has designed redundancy in place to process inbound SMS and app-based push notifications without needing AWS Lambdas. In the case of failure for AWS Lambdas to execute, inbound SMS is received by an application API via a fallback webhook from our telecommunications infrastructure provider \\(Twilio\\). App based push notifications are processed by application level workers that are running in the same tasks that service the rest of the platform and are listening on the same distributed queues that the AWS Lambdas listen on.\n\nEvery application level task also depends on AWS Parameter Store to pick configuration values. The AWS Parameter store is accessed by each application instance at the time of startup to pick up the configuration values and then used for the lifecycle of the application task until the next deployment or instance replacement.\n\n# Impact\n\n**Good news:** Given the fallback logic in place as described above, inbound SMS and user-facing app-based push notifications continued to function without any impact to Lambdas failing. All existing application level tasks continued to operate normally as well since they do not rely on the AWS Parameter Store for normal functioning and only rely on it at startup time. \n\n**Where the impact was felt:** There is typically a limited number of application-level workers running to service badge-count updates and user-facing app-based push notifications. While there was no impact to the user-facing app-based push notifications, badge-count updates were delayed because they are high throughput \\(also provide real time updates to the web\\) given the limited number of workers available to service the badge-count updates. \n\nWhile typically, the Spruce engineering team can easily increase the number of tasks to keep up with and service the badge count updates, in this case that was not possible given that new tasks would not start up given their reliance on AWS Parameter store to pick up their configuration value. So the engineering team decided to stay put and survive with the delayed badge-count updates knowing that the platform was operating normally otherwise.\n\nThe user facing impact was as follows:\n\n* The unread badge count on the Spruce application on smartphones did not update in real time to reflect the right badge count\n* The web-app did not update in real time as it typically does given that the web relies on the badge-count updates to refresh it’s state.\n\n# **Action Items**\n\nWe have 1 solid action item that will improve the overall platform here, which is to reduce dependence on AWS Parameter store by having an in-memory cache \\(Redis\\) as a fallback. Each time an application task starts up, it will write its configuration values to Redis. If a task cannot access AWS Parameter store, it will then fallback to access the values from Redis. We are prioritizing this change given the immediate impact it can bring to the system.\n\nThis change alone will make it so that in the future, if AWS Parameter store is impacted, we can continue to bring up as many application level tasks as we’d like and AWS Lambdas would continue to function normally as well.",
"incident_id": "2tvtp6sb23wf",
"created_at": "2022-09-09T12:03:44.206-07:00",
"updated_at": "2022-09-09T12:29:51.504-07:00",
"display_at": null,
"affected_components": null,
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "q4tywlbsfz44",
"status": "resolved",
"body": "We have started seeing recovery as of 10:34am PT. All badge count updates are processing normally and without delay now.\n\nAWS just confirmed (as of 11:27am PT) what we are seeing, that they too are seeing recovery.\n\nWe will resolve this incident for now since the system is back to operating normally. If you have any questions or concerns please don't hesitate to reach out to us via the Spruce Support conversation in app or support@sprucehealth.com.",
"incident_id": "2tvtp6sb23wf",
"created_at": "2022-09-09T11:34:36.884-07:00",
"updated_at": "2022-09-09T11:34:36.884-07:00",
"display_at": "2022-09-09T11:34:36.884-07:00",
"affected_components": [
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "xf779t173yqn",
"status": "investigating",
"body": "We have identified that Spruce app notifications are actually processing normally without a delay. So app notifications and video calling notifications are working just fine. We had misdiagnosed the issue.\n\nIt is only badge count updates that are delayed at this point. So if you receive a new Page or if you receive a new message, your badge count will not update. But you will see the app notification on your smartphone.",
"incident_id": "2tvtp6sb23wf",
"created_at": "2022-09-09T10:22:48.807-07:00",
"updated_at": "2022-09-09T10:22:48.807-07:00",
"display_at": "2022-09-09T10:22:48.807-07:00",
"affected_components": [
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "rkrljfq7fbfd",
"status": "investigating",
"body": "We are continuing to investigate this issue.",
"incident_id": "2tvtp6sb23wf",
"created_at": "2022-09-09T10:13:13.131-07:00",
"updated_at": "2022-09-09T10:13:13.131-07:00",
"display_at": "2022-09-09T10:13:13.131-07:00",
"affected_components": [
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "dwmnn9n3pp0t",
"status": "investigating",
"body": "Spruce app push notifications are currently delayed due to an issue with our underlying cloud service provider (AWS). We are continuing to monitor the situation and will post an update as soon as we have one to share.\n\nVideo call notifications may also be impacted as a result. So if you are engaging in video calls with your patients, please ask them to have the Spruce app open while waiting for a video call from you. \n\nInbound/outbound calls, secure message exchanges, email, fax and SMS all continue to operate normally.",
"incident_id": "2tvtp6sb23wf",
"created_at": "2022-09-09T10:12:20.179-07:00",
"updated_at": "2022-09-09T10:13:07.483-07:00",
"display_at": "2022-09-09T08:57:20.000-07:00",
"affected_components": [
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "degraded_performance"
},
{
"code": "nftn1q0cs42j",
"name": "Spruce app notifications",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "nftn1q0cs42j",
"name": "Spruce app notifications",
"status": "operational",
"created_at": "2021-12-10T13:25:35.080-08:00",
"updated_at": "2025-10-20T14:35:37.106-07:00",
"position": 8,
"description": "Real time notifications for new messages and video calls.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "th5x2pfgqw0f",
"name": "Spruce inbox failing to load",
"status": "postmortem",
"created_at": "2022-06-01T11:50:00.000-07:00",
"updated_at": "2022-06-02T10:17:11.765-07:00",
"monitoring_at": null,
"resolved_at": "2022-06-01T12:15:59.000-07:00",
"impact": "major",
"shortlink": "https://stspg.io/t36l1xgp4p7t",
"started_at": "2022-06-01T11:50:00.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "nlvhdlxl8bdd",
"status": "postmortem",
"body": "**Analysis**\n\nOne of the instances of a service reached an unhealthy state. Any requests to the instance errored with a timeout given that the instance was unresponsive. It took ~10 minutes for the platform to self-heal and terminate the instance and replace it with a healthy one.\n\nEach client \\(user’s web app or smartphone\\) sends requests to the API layer that round-robins requests to appropriate services to process requests. The round-robin logic is currently not advanced enough to filter out any unhealthy requests, consequently leading to intermittent failed requests until the instance is replaced.\n\n‌\n\n**Action items**\n\n* Explore improving client-side load-balancing logic or using a service mesh to improve observability and reliability of intra-service communication",
"incident_id": "th5x2pfgqw0f",
"created_at": "2022-06-02T10:07:29.386-07:00",
"updated_at": "2022-06-02T10:17:11.763-07:00",
"display_at": null,
"affected_components": null,
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "bzgwrl32rjqt",
"status": "resolved",
"body": "The Spruce Inbox failed to load intermittently for many customers between 11:50am PT and 12:03pm PT. This was due to one of the services reaching an unhealthy state where the one of the instances of the service became unresponsive. Any request from the client to that unhealthy instance resulted in a timeout which to the user resulted in a failed load of the inbox or a particular conversation.\n\nInbound/outbound calling, inbound SMS/Email/Fax/Secure messages were not impacted during this time.\n\nUsers may have been unable to send a message from the Spruce app if their request hit the unhealthy instance, but a subsequent retry likely resolved the issue.",
"incident_id": "th5x2pfgqw0f",
"created_at": "2022-06-02T09:59:59.874-07:00",
"updated_at": "2022-06-02T10:03:51.156-07:00",
"display_at": "2022-06-01T12:15:59.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "yhlk0k51dc2f",
"status": "investigating",
"body": "System automatically resolved itself after identifying the unhealthy task and replacing it with a healthy one",
"incident_id": "th5x2pfgqw0f",
"created_at": "2022-06-02T09:56:59.579-07:00",
"updated_at": "2022-06-02T09:58:11.871-07:00",
"display_at": "2022-06-01T12:03:59.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "d97ytfh4hy9d",
"status": "investigating",
"body": "Spruce inbox is intermittently failing to load for users. Our engineering team is on it and investigating.",
"incident_id": "th5x2pfgqw0f",
"created_at": "2022-06-02T09:56:00.209-07:00",
"updated_at": "2022-06-02T10:00:40.180-07:00",
"display_at": "2022-06-01T11:50:00.000-07:00",
"affected_components": [
{
"code": "nlbprsbttd9h",
"name": "Web App",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "xlz0qxx94v2h",
"name": "Mobile Apps",
"old_status": "operational",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "operational"
}
],
"deliver_notifications": false,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "pfxrfjsvmjqc",
"name": "Call failures and SMS delayed",
"status": "resolved",
"created_at": "2022-03-28T08:01:24.000-07:00",
"updated_at": "2022-03-28T09:49:23.457-07:00",
"monitoring_at": null,
"resolved_at": "2022-03-28T08:45:37.000-07:00",
"impact": "minor",
"shortlink": "https://stspg.io/1y7wggh50405",
"started_at": "2022-03-28T08:01:24.000-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "02zlsm0d42by",
"status": "resolved",
"body": "Our telephony infrastructure provider has reported that all services impacted are now fully operational. Any delayed SMS should now be delivered to inbox. All inbound calls that failed were logged as system failures in the inbox.\n\nWe apologize for the inconvenience this caused, and will be working with our telephony partner to identify the root cause for this issue.",
"incident_id": "pfxrfjsvmjqc",
"created_at": "2022-03-28T09:43:37.871-07:00",
"updated_at": "2022-03-28T09:49:23.438-07:00",
"display_at": "2022-03-28T08:45:37.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "partial_outage",
"new_status": "operational"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "operational"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "partial_outage",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "vlnz88jmg2r9",
"status": "investigating",
"body": "Video calls are also impacted at this time.\n\nOur underlying telephony infrastructure provider is experiencing issues that is resulting in the partial outage on our end for video calls and phone calls.",
"incident_id": "pfxrfjsvmjqc",
"created_at": "2022-03-28T08:51:05.679-07:00",
"updated_at": "2022-03-28T09:49:09.819-07:00",
"display_at": "2022-03-28T08:30:05.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "partial_outage",
"new_status": "partial_outage"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "degraded_performance",
"new_status": "degraded_performance"
},
{
"code": "0pmpkn9dqd64",
"name": "Video Calling",
"old_status": "operational",
"new_status": "partial_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "0jgvdnl8jp28",
"status": "investigating",
"body": "We are investigating failures with inbound and outbound calls.\nInbound SMS is also likely delayed at this time.\n\nThere is no impact to secure messaging, fax or the Spruce inbox at this time.",
"incident_id": "pfxrfjsvmjqc",
"created_at": "2022-03-28T08:31:24.602-07:00",
"updated_at": "2022-03-28T09:49:02.816-07:00",
"display_at": "2022-03-28T08:01:24.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "partial_outage"
},
{
"code": "rfrm421rbt16",
"name": "SMS Routing",
"old_status": "operational",
"new_status": "degraded_performance"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
},
{
"id": "n38l1c46s3vm",
"name": "Issues with inbound calls.",
"status": "resolved",
"created_at": "2022-03-22T09:30:48.381-07:00",
"updated_at": "2022-03-22T12:04:29.436-07:00",
"monitoring_at": "2022-03-22T10:07:17.000-07:00",
"resolved_at": "2022-03-22T12:04:29.417-07:00",
"impact": "major",
"shortlink": "https://stspg.io/j8sz120bclnv",
"started_at": "2022-03-22T09:30:48.376-07:00",
"page_id": "qvq7h8gxchl0",
"incident_updates": [
{
"id": "znkvxfbb2j3y",
"status": "resolved",
"body": "This incident has been resolved.",
"incident_id": "n38l1c46s3vm",
"created_at": "2022-03-22T12:04:29.417-07:00",
"updated_at": "2022-03-22T12:04:29.417-07:00",
"display_at": "2022-03-22T12:04:29.417-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "partial_outage",
"new_status": "operational"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "37tffn7f8tcj",
"status": "monitoring",
"body": "The issue seems to have been resolved by the provider.",
"incident_id": "n38l1c46s3vm",
"created_at": "2022-03-22T10:07:17.542-07:00",
"updated_at": "2022-03-22T10:08:19.040-07:00",
"display_at": "2022-03-22T10:07:17.000-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "partial_outage",
"new_status": "partial_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "bmzm367glzz7",
"status": "investigating",
"body": "Outbound calls are functional at this time. So if a call drops when received, you should be able to place an outgoing call to the phone number to successfully connect with the caller.",
"incident_id": "n38l1c46s3vm",
"created_at": "2022-03-22T09:49:39.745-07:00",
"updated_at": "2022-03-22T09:49:39.745-07:00",
"display_at": "2022-03-22T09:49:39.745-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "partial_outage",
"new_status": "partial_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
},
{
"id": "gr4n20sh5w00",
"status": "investigating",
"body": "Our telephony provider is currently experience issues that are affecting some inbound calls. They are investigating and we will update as we know more.",
"incident_id": "n38l1c46s3vm",
"created_at": "2022-03-22T09:30:48.468-07:00",
"updated_at": "2022-03-22T09:30:48.468-07:00",
"display_at": "2022-03-22T09:30:48.468-07:00",
"affected_components": [
{
"code": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"old_status": "operational",
"new_status": "partial_outage"
}
],
"deliver_notifications": true,
"custom_tweet": null,
"tweet_id": null
}
],
"components": [
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"reminder_intervals": null
}
]
}
Raw
summary.json
{
"page": {
"id": "qvq7h8gxchl0",
"name": "Spruce Health",
"url": "https://status.sprucehealth.com",
"time_zone": "America/Los_Angeles",
"updated_at": "2025-11-10T23:41:11.268-08:00"
},
"components": [
{
"id": "nlbprsbttd9h",
"name": "Web App",
"status": "operational",
"created_at": "2017-03-07T14:46:00.120-08:00",
"updated_at": "2025-10-28T11:58:43.042-07:00",
"position": 1,
"description": "https://app.sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xlz0qxx94v2h",
"name": "Mobile Apps",
"status": "operational",
"created_at": "2017-03-07T14:48:22.872-08:00",
"updated_at": "2025-10-28T11:58:43.056-07:00",
"position": 2,
"description": "iOS \u0026 Android",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "h1mt2skvbqlq",
"name": "Phone Call Routing",
"status": "operational",
"created_at": "2017-03-07T14:48:48.534-08:00",
"updated_at": "2025-10-28T11:58:43.070-07:00",
"position": 3,
"description": "Spruce VoIP, Call Forwarding, Phone Trees and Voicemail",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "5xhfkvp5p5jh",
"name": "Marketing Website",
"status": "operational",
"created_at": "2017-03-07T14:46:00.105-08:00",
"updated_at": "2019-02-13T15:14:40.743-08:00",
"position": 4,
"description": "https://sprucehealth.com",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "6wxpvnqpkbsg",
"name": "Fax",
"status": "operational",
"created_at": "2018-08-14T18:45:20.310-07:00",
"updated_at": "2025-10-28T11:58:43.083-07:00",
"position": 5,
"description": "Inbound/Outbound Digital Fax",
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "rfrm421rbt16",
"name": "SMS Routing",
"status": "operational",
"created_at": "2018-11-05T10:30:16.735-08:00",
"updated_at": "2025-10-28T11:58:43.096-07:00",
"position": 6,
"description": null,
"showcase": true,
"start_date": null,
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "0pmpkn9dqd64",
"name": "Video Calling",
"status": "operational",
"created_at": "2021-12-10T13:25:05.389-08:00",
"updated_at": "2025-10-20T10:37:03.671-07:00",
"position": 7,
"description": "Spruce Video Calling between medical practices and patients.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "nftn1q0cs42j",
"name": "Spruce app notifications",
"status": "operational",
"created_at": "2021-12-10T13:25:35.080-08:00",
"updated_at": "2025-10-20T14:35:37.106-07:00",
"position": 8,
"description": "Real time notifications for new messages and video calls.",
"showcase": true,
"start_date": "2021-09-10",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "xdrmpzwhxky9",
"name": "Integrations",
"status": "operational",
"created_at": "2022-10-12T08:39:52.774-07:00",
"updated_at": "2025-10-20T12:50:00.997-07:00",
"position": 9,
"description": "This component comprises all native integrations between Spruce and external systems, such as Spruce-Elation, Spruce-Hint and Spruce-Stripe integrations.",
"showcase": true,
"start_date": "2022-10-12",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
},
{
"id": "yvdsvc2s20x7",
"name": "API",
"status": "operational",
"created_at": "2023-01-05T12:37:47.464-08:00",
"updated_at": "2025-10-20T12:50:01.014-07:00",
"position": 10,
"description": "https://api.sprucehealth.com",
"showcase": true,
"start_date": "2023-01-05",
"group_id": null,
"page_id": "qvq7h8gxchl0",
"group": false,
"only_show_if_degraded": false
}
],
"incidents": [],
"scheduled_maintenances": [],
"status": { "indicator": "none", "description": "All Systems Operational" }
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment