{"data":{"id":"174740","type":"status_page","attributes":{"company_name":"Nango","company_url":"https://nango.dev","contact_url":"https://nango.dev/slack","logo_url":"https://d1lppblt9t2x15.cloudfront.net/logos/ce52849abb5a99f83f70bd12ff491ed0.png","timezone":"Paris","subdomain":"nango","custom_domain":"status.nango.dev","announcement":null,"logo_remote_url":null,"aggregate_state":"operational","created_at":"2023-11-21T14:45:12.143Z","updated_at":"2026-03-25T12:14:37.015Z"},"relationships":{"sections":{"data":[{"id":"189318","type":"status_page_section"}]},"resources":{"data":[{"id":"8335202","type":"status_page_resource"}]},"status_reports":{"data":[{"id":"841469","type":"status_report"}]}}},"included":[{"id":"189318","type":"status_page_section","attributes":{"name":"","position":0}},{"id":"8335202","type":"status_page_resource","attributes":{"status_page_section_id":189318,"resource_id":1574458,"resource_type":"Monitor","public_name":"Nango Cloud Health","explanation":"Based on monitoring of server/database latency \u0026 syncs/actions/webhooks execution.","position":0,"availability":1.0,"status":"operational","status_history":[{"day":"2025-12-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-12-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-12-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-12-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-12-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2025-12-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-29","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-30","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-01-31","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-05","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-25","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-26","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-27","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-02-28","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-01","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-02","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-03","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-04","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-05","status":"degraded","downtime_duration":4200.0,"maintenance_duration":0},{"day":"2026-03-06","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-07","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-08","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-09","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-10","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-11","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-12","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-13","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-14","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-15","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-16","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-17","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-18","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-19","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-20","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-21","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-22","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-23","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-24","status":"operational","downtime_duration":0,"maintenance_duration":0},{"day":"2026-03-25","status":"operational","downtime_duration":0,"maintenance_duration":0}]}},{"id":"4808579","type":"status_update","attributes":{"message":"Functions recovered. Post mortem: \n\n# Post-Incident Summary\n\n**Date:** 6 March 2026  \n**Impact:** Degraded function execution for actions and webhooks  \n**Status:** Resolved\n\n---\n\n## Summary\n\nA bug in the task scheduling system disabled per-environment concurrency limits, allowing a single tenant to generate an unbounded burst of invocations. At the same time, the tenant’s functions were significantly longer running than typical workloads (120–150 seconds), which caused execution environments to remain occupied for extended periods.\n\nAs the burst of tasks exceeded the rate at which execution environments became available, a backlog formed in the asynchronous invocation queue. This backlog increased the age of queued events and introduced elevated action latency. Eventually, queued tasks began exceeding the expiration limits enforced by the task system and expired before execution.\n\nMonitoring detected the issue through elevated latency metrics, after which the workload was identified and mitigated.\n\n---\n\n## Timeline (CET)\n\n- **Issue began:** 16:00  \n- **Detected by monitoring / on-call paged:** 16:20  \n- **Mitigated:** 17:15  \n\n---\n\n## Root Cause\n\nA bug in the task system disabled per-environment concurrency limits, allowing a single tenant environment to generate an unbounded burst of invocations. The tenant’s functions were also significantly longer running than typical workloads (120–150 seconds), which meant execution environments remained occupied for extended periods and did not recycle quickly.\n\nBecause provisioned concurrency was configured with a low maximum, most of the burst traffic was handled by on-demand capacity. While the function runtime continued scaling additional execution environments, the combination of burst traffic and long-running executions caused capacity to ramp more slowly than the incoming workload required.\n\nThis created a backlog in the asynchronous invocation queue, which increased async event age and action latency. As the backlog grew, queued tasks eventually exceeded the expiration limits enforced by the task system and expired before they could be executed.\n\nDetection was delayed because alerting relies on latency averaged over 15-minute windows.\n\n---\n\n## Resolution\n\n- The source tenant generating the burst workload was identified and incoming traffic was halted.\n- Task system throttling logic was fixed to restore per-environment concurrency limits.\n- After the workload was stopped and backlog drained, function processing returned to normal.\n\nAll systems were fully operational by **17:15 CET**.\n\n---\n\n## Follow-Up Actions\n\n### System safeguards\n- Fix the task system throttling bug to ensure per-environment concurrency limits are always enforced (**completed**).\n\n### Capacity \u0026 scaling\n- Increase provisioned concurrency autoscaling limits to better absorb bursts and reduce cold-start spillover.\n\n### Monitoring \u0026 alerting\n- Add alerts for **AsyncEventAge** to detect queue backlogs earlier.  \n- Add alerts for **ProvisionedConcurrencySpilloverInvocations**.  \n- Add anomaly detection on **actions executed per minute / throughput drops**.","published_at":"2026-03-05T16:15:00.000Z","published_at_timezone":"Paris","notify_subscribers":true,"affected_resources":[{"status_page_resource_id":"8335202","status":"resolved"}]}},{"id":"4808560","type":"status_update","attributes":{"message":"We're implementing a fix that will take effect shortly. Still investigating the root cause.","published_at":"2026-03-05T16:13:00.000Z","published_at_timezone":"Paris","notify_subscribers":true,"affected_resources":[{"status_page_resource_id":"8335202","status":"degraded"}]}},{"id":"4808397","type":"status_update","attributes":{"message":"We are experiencing an issue executing functions on Nango (actions, syncs, and webhooks). We are currently investigating and will provide updates here.","published_at":"2026-03-05T15:05:00.000Z","published_at_timezone":"Paris","notify_subscribers":true,"affected_resources":[{"status_page_resource_id":"8335202","status":"degraded"}]}},{"id":"841469","type":"status_report","attributes":{"title":"Function executions are currently degraded ","report_type":"manual","starts_at":"2026-03-05T15:05:00.000Z","ends_at":null,"affected_resources":[{"status_page_resource_id":"8335202","status":"resolved"}],"aggregate_state":"resolved"},"relationships":{"status_updates":{"data":[{"id":"4808579","type":"status_update"},{"id":"4808560","type":"status_update"},{"id":"4808397","type":"status_update"}]}}}]}