{"page":{"id":"kctbh9vrtdwd","name":"GitHub","url":"https://www.githubstatus.com","time_zone":"Etc/UTC","updated_at":"2026-04-30T13:23:50.177Z"},"incidents":[{"id":"dbypmw7h77l5","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-28T13:59:16.225Z","updated_at":"2026-04-28T17:09:02.466Z","monitoring_at":"2026-04-28T13:59:52.653Z","resolved_at":"2026-04-28T17:09:02.449Z","impact":"minor","shortlink":"https://stspg.io/70pkmpfqz6pc","started_at":"2026-04-28T13:59:16.217Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"fqbr0gx8zt5x","status":"resolved","body":"This incident has been resolved. Thank you for your patience and understanding as we addressed this issue. A detailed root cause analysis will be shared as soon as it is available.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T17:09:02.449Z","updated_at":"2026-04-28T17:09:02.449Z","display_at":"2026-04-28T17:09:02.449Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"4jtcwpljdmf5","status":"monitoring","body":"Actions is operating normally.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T17:08:59.403Z","updated_at":"2026-04-28T17:08:59.403Z","display_at":"2026-04-28T17:08:59.403Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hwlpg9dnbjvd","status":"investigating","body":"Less than 1% of hosted ubuntu-latest runs are delayed. We’re working through remaining steps to restore runner capacity.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T16:36:01.304Z","updated_at":"2026-04-28T16:36:01.304Z","display_at":"2026-04-28T16:36:01.304Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"ph5vyvlg6k7w","status":"investigating","body":"Currently less than 2% of hosted ubuntu-latest and ubuntu-24.04 runs are delayed or failing. We are continuing to monitor for full recovery.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T15:41:48.012Z","updated_at":"2026-04-28T15:41:48.012Z","display_at":"2026-04-28T15:41:48.012Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"srvcmf6v938c","status":"investigating","body":"We've applied a mitigation to unblock running Actions. We're continuing to monitor.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T15:20:16.717Z","updated_at":"2026-04-28T15:20:16.717Z","display_at":"2026-04-28T15:20:16.717Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"lrv23t13wkcv","status":"investigating","body":"We're still investigating the root cause for run start delays and failures for Actions hosted Ubuntu jobs, around 5% of jobs are impacted as of now.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T14:49:49.206Z","updated_at":"2026-04-28T14:49:49.206Z","display_at":"2026-04-28T14:49:49.206Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"9by054y0mswv","status":"investigating","body":"Actions is experiencing degraded performance. We are continuing to investigate.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T14:02:36.720Z","updated_at":"2026-04-28T14:02:36.720Z","display_at":"2026-04-28T14:02:36.720Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"sh63yx8gbnfm","status":"monitoring","body":"Actions is experiencing capacity constraints with hosted ubuntu-latest and ubuntu-24.02, leading to high wait times. Other hosted labels and self-hosted runners are not impacted.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T13:59:52.653Z","updated_at":"2026-04-28T13:59:52.653Z","display_at":"2026-04-28T13:59:52.653Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"z1hcdwfx4t10","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"dbypmw7h77l5","created_at":"2026-04-28T13:59:16.266Z","updated_at":"2026-04-28T13:59:16.266Z","display_at":"2026-04-28T13:59:16.266Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"br0l2tvcx85d","name":"Actions","status":"operational","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2026-04-28T17:08:59.360Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"x69zbgdyfzg0","name":"Incomplete pull request results in repositories","status":"investigating","created_at":"2026-04-28T14:17:34.682Z","updated_at":"2026-04-30T03:49:37.260Z","monitoring_at":null,"resolved_at":null,"impact":"minor","shortlink":"https://stspg.io/87w0vmv3sz1r","started_at":"2026-04-28T14:17:34.674Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"5k01kyhh5h8s","status":"investigating","body":"We have repaired the missing search records for affected Pull Requests and are working to identify and repair records left in a stale state after the recovery.","incident_id":"x69zbgdyfzg0","created_at":"2026-04-30T03:49:37.258Z","updated_at":"2026-04-30T03:49:37.258Z","display_at":"2026-04-30T03:49:37.258Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"75vtfm4ysps7","status":"investigating","body":"We have restored search/indexing functionality for over 99% of impacted pull requests. We are continuing to address the remaining affected pull requests and are reviewing outstanding gaps as part of the restoration process.","incident_id":"x69zbgdyfzg0","created_at":"2026-04-29T22:22:22.208Z","updated_at":"2026-04-29T22:22:22.208Z","display_at":"2026-04-29T22:22:22.208Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"bklk5m37h77k","status":"investigating","body":"Mitigation is in progress, with full recovery of impacted pull request listings expected within approximately 24 hours.","incident_id":"x69zbgdyfzg0","created_at":"2026-04-29T00:40:41.822Z","updated_at":"2026-04-29T00:40:41.822Z","display_at":"2026-04-29T00:40:41.822Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"t0599c1h223b","status":"investigating","body":"We have made an interim mitigation to improve availability for some impacted repositories while reindexing continues, and we are actively monitoring the indexing progress.","incident_id":"x69zbgdyfzg0","created_at":"2026-04-28T22:46:59.005Z","updated_at":"2026-04-28T22:46:59.005Z","display_at":"2026-04-28T22:46:59.005Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"y7fd1szypc3n","status":"investigating","body":"Elastic search reindexing of pull requests is continuing. All data is preserved, but may not be available on pages relying on elasticsearch until the reindex is complete.\u003cbr /\u003e\u003cbr /\u003ePages and APIs that do not rely on elasticsearch, including the GitHub CLI (gh pr list) and API (/repos/{owner}/{repo}/pulls), are not impacted and can be used to retrieve pull request data in the interim.","incident_id":"x69zbgdyfzg0","created_at":"2026-04-28T21:43:17.000Z","updated_at":"2026-04-28T21:43:17.000Z","display_at":"2026-04-28T21:43:17.000Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"0qpymhbwnxv0","status":"investigating","body":"We are actively reindexing the remaining ElasticSearch indexes. Our priority is ensuring correctness and avoiding further impact.  We are taking a measured approach to safely backfill data and will share additional updates as progress continues.","incident_id":"x69zbgdyfzg0","created_at":"2026-04-28T15:58:40.533Z","updated_at":"2026-04-28T15:58:40.533Z","display_at":"2026-04-28T15:58:40.533Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"615cv37kntfg","status":"investigating","body":"After yesterday’s incident, we are investigating cases where /pulls and /repo/pulls pages are not showing all indexed pull requests. This is because our Elasticsearch cluster does not currently contain all indexed documents.\u003cbr /\u003e\u003cbr /\u003eNo pull request data has been lost. As pull requests are updated, they will be reindexed. We are also working on accelerating a full reindex so these pages return complete results again.","incident_id":"x69zbgdyfzg0","created_at":"2026-04-28T14:51:17.509Z","updated_at":"2026-04-28T14:51:17.509Z","display_at":"2026-04-28T14:51:17.509Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"4tmlp24chvf1","status":"investigating","body":"We are investigating reports of degraded performance for Pull Requests","incident_id":"x69zbgdyfzg0","created_at":"2026-04-28T14:17:34.750Z","updated_at":"2026-04-28T14:17:34.750Z","display_at":"2026-04-28T14:17:34.750Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"hhtssxt0f5v2","name":"Pull Requests","status":"degraded_performance","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2026-04-28T15:58:26.154Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"ql942tw29yl6","name":"GitHub search is degraded","status":"resolved","created_at":"2026-04-27T16:31:07.245Z","updated_at":"2026-04-27T22:46:42.345Z","monitoring_at":"2026-04-27T21:33:20.434Z","resolved_at":"2026-04-27T22:46:42.326Z","impact":"critical","shortlink":"https://stspg.io/x5cnngb211t7","started_at":"2026-04-27T16:31:07.236Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"s0g1qphwhmpl","status":"resolved","body":"This incident has been resolved. Thank you for your patience and understanding as we addressed this issue. A detailed root cause analysis will be shared as soon as it is available.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T22:46:42.326Z","updated_at":"2026-04-27T22:46:42.326Z","display_at":"2026-04-27T22:46:42.326Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"rq8twvkjypxh","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T22:44:38.502Z","updated_at":"2026-04-27T22:44:38.502Z","display_at":"2026-04-27T22:44:38.502Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"3c93x80qx5sk","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T22:35:27.164Z","updated_at":"2026-04-27T22:35:27.164Z","display_at":"2026-04-27T22:35:27.164Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"b2yb1lpcbmx7","status":"monitoring","body":"The degradation affecting Actions, Issues, Packages and Pull Requests has been mitigated. We are monitoring to ensure stability.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T21:33:20.434Z","updated_at":"2026-04-27T21:33:20.434Z","display_at":"2026-04-27T21:33:20.434Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"qvs9mt5pqyr6","status":"investigating","body":"We've applied a mitigation and continuing to monitor","incident_id":"ql942tw29yl6","created_at":"2026-04-27T21:32:59.106Z","updated_at":"2026-04-27T21:32:59.106Z","display_at":"2026-04-27T21:32:59.106Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"5h7mh3nf28zj","status":"investigating","body":"Pull Requests is experiencing degraded performance. We are continuing to investigate.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T20:06:40.826Z","updated_at":"2026-04-27T20:06:40.826Z","display_at":"2026-04-27T20:06:40.826Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"major_outage","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"v1z8m9r26sll","status":"investigating","body":"We have identified the source of the additional load causing stress on our ElasticSearch clusters. We have disabled the source of that load and are seeing signs of recovery","incident_id":"ql942tw29yl6","created_at":"2026-04-27T19:50:03.884Z","updated_at":"2026-04-27T19:50:03.884Z","display_at":"2026-04-27T19:50:03.884Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"major_outage","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"78cnm8r63bz2","status":"investigating","body":"Pull Requests is experiencing degraded availability. We are continuing to investigate.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T18:19:33.644Z","updated_at":"2026-04-27T18:19:33.644Z","display_at":"2026-04-27T18:19:33.644Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"qmlflsnzmp19","status":"investigating","body":"We're continuing to see connectivity issues reaching elasticsearch. Impact on downstream services will be intermittent as we find the root cause","incident_id":"ql942tw29yl6","created_at":"2026-04-27T18:17:13.773Z","updated_at":"2026-04-27T18:17:13.773Z","display_at":"2026-04-27T18:17:13.773Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"k160r7drp3dv","status":"investigating","body":"Users are experiencing intermittent failures to view issues, pull requests, projects and Actions workflow runs.\u003cbr /\u003e\u003cbr /\u003eWe are still investigating and attempting mitigations. We will provide further updates.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T17:35:01.416Z","updated_at":"2026-04-27T17:35:01.416Z","display_at":"2026-04-27T17:35:01.416Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vn1g77xczh7h","status":"investigating","body":"Pull Requests is experiencing degraded performance. We are continuing to investigate.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T16:53:22.587Z","updated_at":"2026-04-27T16:53:22.587Z","display_at":"2026-04-27T16:53:22.587Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"fx6y0c206kf3","status":"investigating","body":"Packages is experiencing degraded performance. We are continuing to investigate.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T16:39:04.698Z","updated_at":"2026-04-27T16:39:04.698Z","display_at":"2026-04-27T16:39:04.698Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"n3skwgqx1084","status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T16:36:39.150Z","updated_at":"2026-04-27T16:36:39.150Z","display_at":"2026-04-27T16:36:39.150Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"l78r0dx72c5s","status":"investigating","body":"Customers across GitHub are experiencing failures with searches. Examples include: workflow run failures, projects failing to load, and timed out search requests. This is due to an ongoing infrastructure issue that we have been investigating.","incident_id":"ql942tw29yl6","created_at":"2026-04-27T16:33:05.168Z","updated_at":"2026-04-27T16:33:05.168Z","display_at":"2026-04-27T16:33:05.168Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"t0xbm41f3r6p","status":"investigating","body":"We are investigating reports of degraded performance for Actions","incident_id":"ql942tw29yl6","created_at":"2026-04-27T16:31:07.302Z","updated_at":"2026-04-27T16:31:07.302Z","display_at":"2026-04-27T16:31:07.302Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"kr09ddfgbfsf","name":"Issues","status":"operational","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2026-04-27T21:33:20.375Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"hhtssxt0f5v2","name":"Pull Requests","status":"degraded_performance","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2026-04-28T15:58:26.154Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"br0l2tvcx85d","name":"Actions","status":"operational","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2026-04-28T17:08:59.360Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"st3j38cctv9l","name":"Packages","status":"operational","created_at":"2019-11-13T18:02:40.064Z","updated_at":"2026-04-27T21:33:20.412Z","position":8,"description":"API requests and webhook delivery for GitHub Packages","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"vq183jvj6vrw","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-27T16:48:18.031Z","updated_at":"2026-04-28T18:40:40.145Z","monitoring_at":"2026-04-27T19:01:53.934Z","resolved_at":"2026-04-27T19:02:59.816Z","impact":"minor","shortlink":"https://stspg.io/sqvl4sp1cfgz","started_at":"2026-04-27T16:48:18.023Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"w0pvxfsv859q","status":"resolved","body":"On April 22, 2026 from 18:49 to 19:32 UTC , the Copilot Cloud Agent service began failing during session execution for users running the Agent HQ Codex agent. Codex agent sessions failed to start for all entry points (issue assignment, @copilot comment mentions). 0.5% of total Copilot Cloud Agent jobs were impacted (~2,000 failed jobs). Copilot and other agent sessions were unaffected.\u003cbr /\u003e\u003cbr /\u003eThis was caused by a model resolution mismatch in Codex agent sessions, resulting in an incompatible model being used at runtime. A mitigation was deployed to select a stable default model for Codex agent sessions.\u003cbr /\u003e\u003cbr /\u003eWe are working to harden the underlying model-resolution path so it correctly scopes to the requesting agent's supported models to prevent similar failure mode in the future.","incident_id":"vq183jvj6vrw","created_at":"2026-04-27T19:02:59.816Z","updated_at":"2026-04-28T18:40:40.140Z","display_at":"2026-04-27T19:02:59.816Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"l8zkkfjs3dkp","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"vq183jvj6vrw","created_at":"2026-04-27T19:01:53.934Z","updated_at":"2026-04-27T19:01:53.934Z","display_at":"2026-04-27T19:01:53.934Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"h4lgtqk4lz12","status":"investigating","body":"We've found the issue and are working on deploying a solution to get Codex agent runs working again.","incident_id":"vq183jvj6vrw","created_at":"2026-04-27T17:26:59.299Z","updated_at":"2026-04-27T17:26:59.299Z","display_at":"2026-04-27T17:26:59.299Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"gqz10fd12m95","status":"investigating","body":"Copilot Cloud Agent (CCA) jobs using the Codex agent are failing after starting. To avoid this issue, please choose a different agent. We are investigating the cause and working towards remediation","incident_id":"vq183jvj6vrw","created_at":"2026-04-27T17:01:56.904Z","updated_at":"2026-04-27T17:01:56.904Z","display_at":"2026-04-27T17:01:56.904Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"31wwrf034cwk","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"vq183jvj6vrw","created_at":"2026-04-27T16:48:18.075Z","updated_at":"2026-04-27T16:48:18.075Z","display_at":"2026-04-27T16:48:18.075Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"3dqg0nhwnxs2","name":"Delays with Actions Jobs for Larger Runners using VNet Injection in the East US region","status":"resolved","created_at":"2026-04-24T19:02:39.061Z","updated_at":"2026-04-28T01:56:43.326Z","monitoring_at":null,"resolved_at":"2026-04-25T00:36:04.272Z","impact":"minor","shortlink":"https://stspg.io/mcrpvdrwjzf3","started_at":"2026-04-24T19:02:39.054Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"xyz55tklj4hd","status":"resolved","body":"On April 24, 2026, from approximately 11:39 UTC to April 25, 2026 at 00:15 UTC, GitHub Actions experienced delays and timeouts for Larger Hosted Runner jobs using VNet injection in the East US region without a failover region configured. Standard and Self-hosted runners were not impacted. This was caused by backend failures in our compute provider’s provisioning, scaling, and update operations for VMs in the East US region and mitigated by a rollback across all affected Availability Zones. More detail is available at https://azure.status.microsoft/en-us/status/history/?trackingId=5GP8-W0G.\u003cbr /\u003e\u003cbr /\u003eWe are working to improve the reliability of our annotations for jobs impacted by regional issues and are adding system log notifications as an additional customer communication channel alongside annotations.\u003cbr /\u003e\u003cbr /\u003eVNet Failover is also now in public preview, allowing customers to evacuate Larger Hosted Runners using VNet injection in cases like this.","incident_id":"3dqg0nhwnxs2","created_at":"2026-04-25T00:36:04.272Z","updated_at":"2026-04-28T01:56:43.320Z","display_at":"2026-04-25T00:36:04.272Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"2xk78skvj6b4","status":"investigating","body":"This is related to the public impact, \"Multiservice impact for Azure Workloads in East US\" shared at https://azure.status.microsoft/","incident_id":"3dqg0nhwnxs2","created_at":"2026-04-24T19:14:51.753Z","updated_at":"2026-04-24T19:14:51.753Z","display_at":"2026-04-24T19:14:51.753Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"jqys36s3s5mf","status":"investigating","body":"We are investigating reports of degraded performance for Larger Runners with vnet injection in East US and we are working with our service provider on mitigation.","incident_id":"3dqg0nhwnxs2","created_at":"2026-04-24T19:09:12.236Z","updated_at":"2026-04-24T19:09:12.236Z","display_at":"2026-04-24T19:09:12.236Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"rdtptlys3mx4","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"3dqg0nhwnxs2","created_at":"2026-04-24T19:02:39.102Z","updated_at":"2026-04-24T19:02:39.102Z","display_at":"2026-04-24T19:02:39.102Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"zsg1lk7w13cf","name":"Incident with Pull Requests","status":"resolved","created_at":"2026-04-23T19:50:51.291Z","updated_at":"2026-04-26T18:15:30.236Z","monitoring_at":null,"resolved_at":"2026-04-23T21:43:26.000Z","impact":"minor","shortlink":"https://stspg.io/byp3p9jymrpl","started_at":"2026-04-23T21:43:26.000Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"2ncnkp2q0k1b","status":"resolved","body":"On April 23, 2026, between 16:05 UTC and 20:43 UTC, the Pull Requests service experienced a regression affecting merge queue operations. PRs merged via merge queue using the squash merge method produced incorrect merge commits when the merge group contained more than one PR. In affected cases, changes from previously merged PRs and prior commits were inadvertently reverted by subsequent merges.\u003cbr /\u003e\u003cbr /\u003eDuring the impact window 2,092 pull requests were affected. The issue did not affect pull requests merged outside of merge queue, nor merge queue groups using the merge or rebase methods.\u003cbr /\u003eIt took approximately 3 hours and 33 minutes to identify the issue. The change completed deployment at approximately 16:05 UTC, and we became aware at 19:38 UTC following an increase in customer support inquiries. Because the issue affected merge commit correctness rather than availability, it was not detected by existing automated monitoring and was identified through customer reports.\u003cbr /\u003e\u003cbr /\u003eThe regression was introduced by a new code path that adjusted merge base computation for merge queue ref updates. This code path was intended to be gated behind a feature flag for an unreleased feature, but the gating was incomplete.\u003cbr /\u003e\u003cbr /\u003eAs a result, the new behavior was inadvertently applied to squash merge groups, producing an incorrect three-way merge. This caused subsequent squash merges to revert changes from earlier pull requests and, in some cases, changes between their starting points.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by reverting the code change and force-deploying the fix across all environments. After resolution, we identified affected repositories and sent targeted remediation instructions to repository administrators with step-by-step recovery guidance.\u003cbr /\u003e\u003cbr /\u003eThe regression was not identified during internal validation. Existing test coverage primarily exercised single-PR merge queue groups, which did not exhibit the faulty base-reference calculation. Because automated checks did not validate merge correctness for multi-PR squash groups, the defect surfaced only in production.\u003cbr /\u003e\u003cbr /\u003eTo prevent recurrence, GitHub is expanding test coverage for merge correctness validation. We are broadening automated coverage for merge queue operations, including regression checks that validate resulting Git contents across supported configurations, so issues affecting merge correctness are caught before reaching production.\u003cbr /\u003e\u003cbr /\u003eWe are committed to ensuring the correctness and reliability of merge queue operations. These actions will reduce the risk of similar regressions and improve confidence in future changes to the Pull Requests service.","incident_id":"zsg1lk7w13cf","created_at":"2026-04-23T21:43:26.267Z","updated_at":"2026-04-26T18:15:30.231Z","display_at":"2026-04-23T21:43:26.000Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"0jst7wrjww50","status":"investigating","body":"We have resolved a regression present when using merge queue with either squash merges or rebases. If you use merge queue in this configuration, some pull requests may have been merged incorrectly between 2026-04-23 16:05-20:43 UTC. \u003cbr /\u003e\u003cbr /\u003eThis behavior is still present in GitHub Enterprise Cloud with Data Residency, and we are rolling out the same fix.","incident_id":"zsg1lk7w13cf","created_at":"2026-04-23T21:18:15.963Z","updated_at":"2026-04-23T21:18:15.963Z","display_at":"2026-04-23T21:18:15.963Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"v2y8yr81mv57","status":"investigating","body":"Pull Requests is operating normally.","incident_id":"zsg1lk7w13cf","created_at":"2026-04-23T20:47:01.210Z","updated_at":"2026-04-23T20:47:01.210Z","display_at":"2026-04-23T20:47:01.210Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"7wcc3wp5shr4","status":"investigating","body":"We have identified a regression in merge queue behavior present when squash merging or rebasing. We have identified the root-cause and are in the process of reverting the change.","incident_id":"zsg1lk7w13cf","created_at":"2026-04-23T19:58:14.939Z","updated_at":"2026-04-23T19:58:14.939Z","display_at":"2026-04-23T19:58:14.939Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"3rcv77hss4sf","status":"investigating","body":"We are investigating reports of degraded performance for Pull Requests","incident_id":"zsg1lk7w13cf","created_at":"2026-04-23T19:50:51.355Z","updated_at":"2026-04-23T19:50:51.355Z","display_at":"2026-04-23T19:50:51.355Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"hhtssxt0f5v2","name":"Pull Requests","status":"degraded_performance","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2026-04-28T15:58:26.154Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"40f3r1jvjwl2","name":"Disruption with users unable to start Claude and Codex agent task from the web","status":"resolved","created_at":"2026-04-23T19:28:42.976Z","updated_at":"2026-04-28T16:37:50.885Z","monitoring_at":null,"resolved_at":"2026-04-23T19:42:25.277Z","impact":"minor","shortlink":"https://stspg.io/nwmzvw3twyyg","started_at":"2026-04-23T19:28:42.969Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"l7kt57215rwq","status":"resolved","body":"Between 18:45 and 19:42 UTC on April 23, users were unable to start new agent tasks using either Claude or Codex agent on github.com. This was caused by a code change to how Copilot mission control routes task creation requests. Ongoing agent tasks and other Copilot agent features were not affected. We mitigated the impact by reverting the breaking change. We are adding extra monitoring and integration test coverage for the task creation path to prevent future recurrence.","incident_id":"40f3r1jvjwl2","created_at":"2026-04-23T19:42:25.277Z","updated_at":"2026-04-28T16:37:50.879Z","display_at":"2026-04-23T19:42:25.277Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"47mwhxzvdbk1","status":"investigating","body":"We have identified the root cause of the issue and are working on mitigation.","incident_id":"40f3r1jvjwl2","created_at":"2026-04-23T19:33:27.279Z","updated_at":"2026-04-23T19:33:27.279Z","display_at":"2026-04-23T19:33:27.279Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"qfdfcn3mv3th","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"40f3r1jvjwl2","created_at":"2026-04-23T19:28:43.005Z","updated_at":"2026-04-23T19:28:43.005Z","display_at":"2026-04-23T19:28:43.005Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"myrbk7jvvs6p","name":"Incident with multiple GitHub services","status":"resolved","created_at":"2026-04-23T16:12:47.744Z","updated_at":"2026-04-23T17:30:49.733Z","monitoring_at":null,"resolved_at":"2026-04-23T17:30:49.716Z","impact":"critical","shortlink":"https://stspg.io/1wm8c7d53454","started_at":"2026-04-23T16:12:47.737Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"gg1bvv6q3v4g","status":"resolved","body":"This incident has been resolved. Thank you for your patience and understanding as we addressed this issue. A detailed root cause analysis will be shared as soon as it is available.","incident_id":"myrbk7jvvs6p","created_at":"2026-04-23T17:30:49.716Z","updated_at":"2026-04-23T17:30:49.716Z","display_at":"2026-04-23T17:30:49.716Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"4yctlwq6n1nt","status":"investigating","body":"Webhooks is operating normally.","incident_id":"myrbk7jvvs6p","created_at":"2026-04-23T17:10:29.671Z","updated_at":"2026-04-23T17:10:29.671Z","display_at":"2026-04-23T17:10:29.671Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"degraded_performance","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"lk5hklw7dgcl","status":"investigating","body":"Many services are mitigated and are validating the remaining services.","incident_id":"myrbk7jvvs6p","created_at":"2026-04-23T17:04:30.623Z","updated_at":"2026-04-23T17:04:30.623Z","display_at":"2026-04-23T17:04:30.623Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"qd6745fhn70j","status":"investigating","body":"The degradation affecting Actions and Copilot has been mitigated. We are monitoring to ensure stability.","incident_id":"myrbk7jvvs6p","created_at":"2026-04-23T17:03:00.933Z","updated_at":"2026-04-23T17:03:00.933Z","display_at":"2026-04-23T17:03:00.933Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"major_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"bm9sgtpch4br","status":"investigating","body":"We have identified the root problem and are working on mitigation.","incident_id":"myrbk7jvvs6p","created_at":"2026-04-23T16:52:50.440Z","updated_at":"2026-04-23T16:52:50.440Z","display_at":"2026-04-23T16:52:50.440Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"major_outage","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"g3cs1mmsg359","status":"investigating","body":"Actions is experiencing degraded performance. We are continuing to investigate.","incident_id":"myrbk7jvvs6p","created_at":"2026-04-23T16:34:19.448Z","updated_at":"2026-04-23T16:34:19.448Z","display_at":"2026-04-23T16:34:19.448Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"degraded_performance"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"major_outage","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hd70gvx1ftw8","status":"investigating","body":"We are investigating multiple unavailable services.","incident_id":"myrbk7jvvs6p","created_at":"2026-04-23T16:19:33.310Z","updated_at":"2026-04-23T16:19:33.310Z","display_at":"2026-04-23T16:19:33.310Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"major_outage","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"1tgs7ngcvvs8","status":"investigating","body":"We are investigating reports of degraded availability for Copilot and Webhooks","incident_id":"myrbk7jvvs6p","created_at":"2026-04-23T16:12:47.817Z","updated_at":"2026-04-23T16:12:47.817Z","display_at":"2026-04-23T16:12:47.817Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"degraded_performance"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"4230lsnqdsld","name":"Webhooks","status":"operational","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2026-04-23T17:10:29.626Z","position":2,"description":"Real time HTTP callbacks of user-generated and system events","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"br0l2tvcx85d","name":"Actions","status":"operational","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2026-04-28T17:08:59.360Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"pjmpxvq2cmr2","name":"Copilot","status":"operational","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2026-04-23T17:03:00.903Z","position":10,"description":null,"showcase":true,"start_date":"2022-06-21","group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"3f30ycplyr7l","name":"Investigating errors on GitHub","status":"resolved","created_at":"2026-04-23T14:40:17.981Z","updated_at":"2026-04-23T15:18:41.383Z","monitoring_at":"2026-04-23T15:02:29.466Z","resolved_at":"2026-04-23T15:18:41.367Z","impact":"minor","shortlink":"https://stspg.io/v510xk27crlr","started_at":"2026-04-23T14:40:17.973Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"2788m0qgc1gh","status":"resolved","body":"This incident has been resolved. Thank you for your patience and understanding as we addressed this issue. A detailed root cause analysis will be shared as soon as it is available.","incident_id":"3f30ycplyr7l","created_at":"2026-04-23T15:18:41.367Z","updated_at":"2026-04-23T15:18:41.367Z","display_at":"2026-04-23T15:18:41.367Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"8vbcqps543xj","status":"monitoring","body":"The degradation affecting Actions, Codespaces, Copilot and Packages has been mitigated. We are monitoring to ensure stability.","incident_id":"3f30ycplyr7l","created_at":"2026-04-23T15:02:29.466Z","updated_at":"2026-04-23T15:02:29.466Z","display_at":"2026-04-23T15:02:29.466Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"operational"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"degraded_performance","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"degraded_performance","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"yqmw636ch4t0","status":"investigating","body":"A mitigation was applied and services have recovered.  Actions is working through queued work before fully recovering.","incident_id":"3f30ycplyr7l","created_at":"2026-04-23T15:02:15.455Z","updated_at":"2026-04-23T15:02:15.455Z","display_at":"2026-04-23T15:02:15.455Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"0pts64cdhq1h","status":"investigating","body":"Users are experiencing errors loading various web pages on github.com. Actions and Copilot Cloud Agent runs will be delayed.","incident_id":"3f30ycplyr7l","created_at":"2026-04-23T14:51:12.193Z","updated_at":"2026-04-23T14:51:12.193Z","display_at":"2026-04-23T14:51:12.193Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"r2cdvdgc8cs5","status":"investigating","body":"Copilot is experiencing degraded performance. We are continuing to investigate.","incident_id":"3f30ycplyr7l","created_at":"2026-04-23T14:44:55.905Z","updated_at":"2026-04-23T14:44:55.905Z","display_at":"2026-04-23T14:44:55.905Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"fcwrpfdtv93t","status":"investigating","body":"Codespaces is experiencing degraded performance. We are continuing to investigate.","incident_id":"3f30ycplyr7l","created_at":"2026-04-23T14:42:43.062Z","updated_at":"2026-04-23T14:42:43.062Z","display_at":"2026-04-23T14:42:43.062Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"d44fg50v8myq","status":"investigating","body":"Packages is experiencing degraded performance. We are continuing to investigate.","incident_id":"3f30ycplyr7l","created_at":"2026-04-23T14:41:32.959Z","updated_at":"2026-04-23T14:41:32.959Z","display_at":"2026-04-23T14:41:32.959Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"degraded_performance","new_status":"degraded_performance"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"pgm5z80ks1cx","status":"investigating","body":"We are investigating reports of degraded performance for Actions","incident_id":"3f30ycplyr7l","created_at":"2026-04-23T14:40:18.054Z","updated_at":"2026-04-23T14:40:18.054Z","display_at":"2026-04-23T14:40:18.054Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"br0l2tvcx85d","name":"Actions","status":"operational","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2026-04-28T17:08:59.360Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"st3j38cctv9l","name":"Packages","status":"operational","created_at":"2019-11-13T18:02:40.064Z","updated_at":"2026-04-27T21:33:20.412Z","position":8,"description":"API requests and webhook delivery for GitHub Packages","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"pjmpxvq2cmr2","name":"Copilot","status":"operational","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2026-04-23T17:03:00.903Z","position":10,"description":null,"showcase":true,"start_date":"2022-06-21","group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"h2ftsgbw7kmk","name":"Codespaces","status":"operational","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2026-04-23T15:02:29.440Z","position":11,"description":"Orchestration and Compute for GitHub Codespaces","showcase":true,"start_date":"2021-08-11","group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"c53mty24vwg8","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-22T19:53:49.917Z","updated_at":"2026-04-27T20:53:58.069Z","monitoring_at":null,"resolved_at":"2026-04-22T22:43:59.100Z","impact":"minor","shortlink":"https://stspg.io/xmhs8h2d6zlj","started_at":"2026-04-22T19:53:49.911Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"f6hm4yfr14gd","status":"resolved","body":"On April 22, 2026, between 09:00 UTC and 22:05 UTC, the Copilot coding agent and pull request comment event processing were degraded. During this period, approximately 0.5% of total pull request and issue comments mentioned @copilot (~23,000 invocations), explicitly requested work from the Copilot coding agent but were not acted upon.\u003cbr /\u003e\u003cbr /\u003eCreating, viewing, and replying to pull request comments was unaffected, and other Copilot\u003cbr /\u003efunctionality continued to operate normally. The impact was limited to @copilot mentions on pull request comments not triggering Copilot coding agent runs, and to some downstream systems not receiving new pull request comment events during the impact window.\u003cbr /\u003e\u003cbr /\u003eThe cause was a serialization error that prevented pull request comment events from being published to downstream consumers, including the Copilot coding agent. This was related to the same class of issue as incident #4295 on April 20, affecting a another event type.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by deploying a fix that restored event publishing, after which the Copilot coding agent and other downstream consumers resumed processing pull request comment events normally.\u003cbr /\u003e\u003cbr /\u003eWe are working to complete our audit of related event schemas, migrate remaining consumers to use\u003cbr /\u003ethe updated identifier fields, and improve monitoring to detect drops in publishing on critical event topics, to reduce our time to detection and mitigation of issues like this one in the future.","incident_id":"c53mty24vwg8","created_at":"2026-04-22T22:43:59.100Z","updated_at":"2026-04-27T20:53:58.065Z","display_at":"2026-04-22T22:43:59.100Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"2w83l8sd9dr8","status":"investigating","body":"We have identified the root cause of the disruption affecting Copilot Coding Agent and Issues. A fix is being deployed.","incident_id":"c53mty24vwg8","created_at":"2026-04-22T22:09:08.662Z","updated_at":"2026-04-22T22:09:08.662Z","display_at":"2026-04-22T22:09:08.662Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"xy8r4sd0bg4l","status":"investigating","body":"We have identified the root cause of the disruption affecting Copilot Coding Agent and Issues. Copilot @-mentions on pull requests are not being processed, and some issue-related functionality may be degraded. A fix has been developed and is being applied. \u003cbr /\u003e","incident_id":"c53mty24vwg8","created_at":"2026-04-22T20:37:21.959Z","updated_at":"2026-04-22T20:37:21.959Z","display_at":"2026-04-22T20:37:21.959Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"ychyl0ylyyl5","status":"investigating","body":"Copilot @-mentions on pull requests are currently not being processed by Copilot Cloud Agent. We have found the issue and are investigating remediations.","incident_id":"c53mty24vwg8","created_at":"2026-04-22T20:02:21.774Z","updated_at":"2026-04-22T20:02:21.774Z","display_at":"2026-04-22T20:02:21.774Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"cjkpn2j5mfk7","status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","incident_id":"c53mty24vwg8","created_at":"2026-04-22T19:55:47.835Z","updated_at":"2026-04-22T19:55:47.835Z","display_at":"2026-04-22T19:55:47.835Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vpfxqqnch2xy","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"c53mty24vwg8","created_at":"2026-04-22T19:53:49.957Z","updated_at":"2026-04-22T19:53:49.957Z","display_at":"2026-04-22T19:53:49.957Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"kr09ddfgbfsf","name":"Issues","status":"operational","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2026-04-27T21:33:20.375Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"8790jgb3ky6t","name":"Disruption with Copilot chat and Copilot Coding Agent","status":"resolved","created_at":"2026-04-22T15:35:45.298Z","updated_at":"2026-04-27T21:28:07.781Z","monitoring_at":null,"resolved_at":"2026-04-22T19:18:26.916Z","impact":"critical","shortlink":"https://stspg.io/0m0g0r5htdxy","started_at":"2026-04-22T15:35:45.288Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"p023q6brdgtv","status":"resolved","body":"On April 22, 2026, between 15:16 UTC and 19:18 UTC, users experienced errors when interacting with Copilot Chat on github.com and Copilot Cloud Agent. During this time, users were unable to use Copilot Chat or Copilot Cloud Agent. Copilot Memory (in preview) was not available to Copilot agent sessions during this time. The issue was caused by an infrastructure configuration change that resulted in connectivity issues with our databases. The team identified the cause and restored connectivity to the database. Copilot Chat and Cloud Agent for github.com were restored by 18:16 UTC. Remaining regional deployments were restored incrementally, with full resolution at 19:18 UTC. We have taken steps to prevent similar infrastructure changes from causing these kinds of database operations in the future.","incident_id":"8790jgb3ky6t","created_at":"2026-04-22T19:18:26.916Z","updated_at":"2026-04-27T21:28:07.776Z","display_at":"2026-04-22T19:18:26.916Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"ykx871mdhp92","status":"investigating","body":"Copilot cloud agent and chat are mitigated for github.com.","incident_id":"8790jgb3ky6t","created_at":"2026-04-22T18:05:40.212Z","updated_at":"2026-04-22T18:05:40.212Z","display_at":"2026-04-22T18:05:40.212Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"ky6v9n5snm6k","status":"investigating","body":"We are now seeing recovery for Copilot cloud agent.","incident_id":"8790jgb3ky6t","created_at":"2026-04-22T17:49:20.646Z","updated_at":"2026-04-22T17:49:20.646Z","display_at":"2026-04-22T17:49:20.646Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"w25zwh8blmfp","status":"investigating","body":"Mitigation is progressing for Copilot chat and cloud agent recovery.","incident_id":"8790jgb3ky6t","created_at":"2026-04-22T17:40:07.509Z","updated_at":"2026-04-22T17:40:07.509Z","display_at":"2026-04-22T17:40:07.509Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hns942sc3gx0","status":"investigating","body":"Mitigation is progressing for Copilot chat and cloud agent.","incident_id":"8790jgb3ky6t","created_at":"2026-04-22T16:58:36.975Z","updated_at":"2026-04-22T16:58:36.975Z","display_at":"2026-04-22T16:58:36.975Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"k6l5vww51drc","status":"investigating","body":"We continue to work on mitigation for Copilot chat and cloud agent.","incident_id":"8790jgb3ky6t","created_at":"2026-04-22T16:24:52.308Z","updated_at":"2026-04-22T16:24:52.308Z","display_at":"2026-04-22T16:24:52.308Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"45f1ph7n5073","status":"investigating","body":"We are aware of users seeing errors interacting with Copilot chat on github.com and Copilot cloud agent. We have identified the cause and are investigating remediations.","incident_id":"8790jgb3ky6t","created_at":"2026-04-22T15:43:55.210Z","updated_at":"2026-04-22T15:43:55.210Z","display_at":"2026-04-22T15:43:55.210Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"k6dtnvjg1cty","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"8790jgb3ky6t","created_at":"2026-04-22T15:35:45.369Z","updated_at":"2026-04-22T15:35:45.369Z","display_at":"2026-04-22T15:35:45.369Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"b4235qyfzx2z","name":"Disruption with projects service","status":"resolved","created_at":"2026-04-21T15:03:21.160Z","updated_at":"2026-04-27T23:57:33.300Z","monitoring_at":"2026-04-21T21:15:06.169Z","resolved_at":"2026-04-22T01:24:13.877Z","impact":"minor","shortlink":"https://stspg.io/z7byhh6vyyrb","started_at":"2026-04-21T15:03:21.155Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"69q7hw84t17m","status":"resolved","body":"On April 21, 2026, between 13:35 UTC and 01:24 UTC the following day the projects service was degraded. During this time period, projects may have been out of sync and users may have experienced delays in changes to projects and their items. Delays in reflected changes peaked at approximately 45 minutes. The delays were caused by serialization errors that failed events and triggered a flood of resyncs, overloading our event processing layers.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by speeding up processing time for incoming changes and otherwise waiting for all changes to be processed.\u003cbr /\u003e\u003cbr /\u003eWe are working to increase our capacity for processing updates to projects to reduce our time to mitigation of issues like this one in the future.","incident_id":"b4235qyfzx2z","created_at":"2026-04-22T01:24:13.877Z","updated_at":"2026-04-27T23:57:33.295Z","display_at":"2026-04-22T01:24:13.877Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"1k4y3wznwvwq","status":"monitoring","body":"The issue remains mitigated. Users may still experience small delays in changes to projects while we process the backlog of events. We expect a full recovery in approximately two hours.","incident_id":"b4235qyfzx2z","created_at":"2026-04-22T00:00:49.877Z","updated_at":"2026-04-22T00:00:49.877Z","display_at":"2026-04-22T00:00:49.877Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"74vhjxw0rgz0","status":"monitoring","body":"The issue remains mitigated. Users may still experience delays in changes to projects while we process the backlog of events. We expect a full recovery in approximately three hours.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T22:49:52.599Z","updated_at":"2026-04-21T22:49:52.599Z","display_at":"2026-04-21T22:49:52.599Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"wfl8qmmjdtk0","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T21:15:06.169Z","updated_at":"2026-04-21T21:15:06.169Z","display_at":"2026-04-21T21:15:06.169Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"8mk01fjhhfr4","status":"investigating","body":"Recovery from the delays affecting GitHub Projects continues to progress. We have deployed additional mitigations that are accelerating processing of the backlog. Users may still experience delays where changes to projects are not reflected immediately. We expect full recovery within approximately six hours.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T19:41:18.904Z","updated_at":"2026-04-21T19:41:18.904Z","display_at":"2026-04-21T19:41:18.904Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"r5t4s0gxz3sj","status":"investigating","body":"The queues are continuing to decrease and we are working to accelerate the rate of processing through the queues.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T17:21:33.098Z","updated_at":"2026-04-21T17:21:33.098Z","display_at":"2026-04-21T17:21:33.098Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"7l78ht3j40kz","status":"investigating","body":"The mitigation is deployed and we are seeing recorvery in the queues and will provide an update as to when full recovery will be realized.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T16:45:22.676Z","updated_at":"2026-04-21T16:45:22.676Z","display_at":"2026-04-21T16:45:22.676Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"43b5p6jnyth5","status":"investigating","body":"We are deploying a fix to relieve the queue of delayed data. Some users may still experience delays with GitHub Projects where changes are not reflected immediately as remaining backlogs are processed.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T16:18:24.208Z","updated_at":"2026-04-21T16:18:24.208Z","display_at":"2026-04-21T16:18:24.208Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"0nh611nb7x4w","status":"investigating","body":"We continue to investigate delays with GitHub Projects where changes may not be reflected immediately. Our team has identified the cause and applied mitigations to address the issue. We are seeing initial signs of recovery, though some delays may persist as the system works through a backlog of pending updates.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T15:42:20.758Z","updated_at":"2026-04-21T15:42:20.758Z","display_at":"2026-04-21T15:42:20.758Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"n0g2lcs5dxw1","status":"investigating","body":"We are investigating reports of delays with GitHub Projects. Users may notice that changes made to projects are not reflected immediately. Our team has identified the source of the delays and is actively working to resolve the issue.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T15:07:30.717Z","updated_at":"2026-04-21T15:07:30.717Z","display_at":"2026-04-21T15:07:30.717Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"07189frbn6d0","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"b4235qyfzx2z","created_at":"2026-04-21T15:03:21.193Z","updated_at":"2026-04-21T15:03:21.193Z","display_at":"2026-04-21T15:03:21.193Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"m7pgzw1wlfq7","name":"Partial degradation for code scanning default setup and for code quality","status":"resolved","created_at":"2026-04-20T13:28:48.571Z","updated_at":"2026-04-24T18:44:43.056Z","monitoring_at":"2026-04-20T18:20:38.322Z","resolved_at":"2026-04-21T05:04:56.242Z","impact":"major","shortlink":"https://stspg.io/qxk1qwhj9xdd","started_at":"2026-04-20T13:28:48.564Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"5kkqgm8fnjq6","status":"resolved","body":"On April 20, 2026 between 10:28 UTC and 15:04 UTC GitHub experienced degraded service for code scanning default setup, code quality, and project boards. Repair of affected project boards additionally lasted until April 21, 05:04 UTC \u003cbr /\u003e\u003cbr /\u003eDuring this time, code scanning default setup and code quality analyses were not triggered on newly opened pull requests. Additionally, newly created issues were not appearing on project boards. \u003cbr /\u003e\u003cbr /\u003eThe cause was a serialization error that prevented proper triggering of code scanning, code quality analyses, and project board updates. \u003cbr /\u003e\u003cbr /\u003eWe mitigated the issue by deploying a fix, restoring event publishing for code scanning and code quality. For project boards, an additional code change was deployed to update event consumers, followed by a reindex of affected project items. \u003cbr /\u003e\u003cbr /\u003eWe are working to prevent recurrence by strengthening our schema validations and improving monitoring for drops in publishing on critical hydro topics.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-21T05:04:56.242Z","updated_at":"2026-04-24T18:44:43.050Z","display_at":"2026-04-21T05:04:56.242Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"lr40nzncgwl1","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-21T04:18:47.451Z","updated_at":"2026-04-21T04:18:47.451Z","display_at":"2026-04-21T04:18:47.451Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"kl8ysvbfknp1","status":"monitoring","body":"The issue remains mitigated. Issues that were linked to projects during the incident may take approximately three more hours to render correctly while we complete a re-index.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-21T03:10:52.364Z","updated_at":"2026-04-21T03:10:52.364Z","display_at":"2026-04-21T03:10:52.364Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"k8kzvxlbmwdx","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T21:36:26.143Z","updated_at":"2026-04-20T21:36:26.143Z","display_at":"2026-04-20T21:36:26.143Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"7flyfltgrn38","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T18:21:15.361Z","updated_at":"2026-04-20T18:21:15.361Z","display_at":"2026-04-20T18:21:15.361Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"bb4qc8cj7c17","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T18:20:38.322Z","updated_at":"2026-04-20T18:20:38.322Z","display_at":"2026-04-20T18:20:38.322Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"m4ml137bb39p","status":"investigating","body":"The issue has been mitigated. Newly created issues linked to projects should now function as expected. Issues that were linked to projects during the incident may take approximately five hours to render correctly while we complete a re-index.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T18:20:26.422Z","updated_at":"2026-04-20T18:20:26.422Z","display_at":"2026-04-20T18:20:26.422Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"gvhhxjlk296r","status":"investigating","body":"A deployment to fix this issue of new issues not showing up in projects is underway.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T18:08:33.678Z","updated_at":"2026-04-20T18:08:33.678Z","display_at":"2026-04-20T18:08:33.678Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"wvjpb5pqgvzb","status":"investigating","body":"We continue to work on mitigation regarding new issues not showing on project boards.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T17:32:37.159Z","updated_at":"2026-04-20T17:32:37.159Z","display_at":"2026-04-20T17:32:37.159Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vy8xt95r50wc","status":"investigating","body":"We continue to work on mitigation regarding new issues not showing on project boards.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T16:48:35.387Z","updated_at":"2026-04-20T16:48:35.387Z","display_at":"2026-04-20T16:48:35.387Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"q0x4hc3qd1q9","status":"investigating","body":"Code scanning default setup and Code Quality triggers are back up and running. PRs not processed before or during this incident will require a new push to trigger code scanning or code quality analysis.\u003cbr /\u003e\u003cbr /\u003eWe are seeing problems with new issues not showing on project boards and are working on mitigation.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T16:16:24.957Z","updated_at":"2026-04-20T16:16:24.957Z","display_at":"2026-04-20T16:16:24.957Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"z6bqg33rdx9z","status":"investigating","body":"We are continuing to work on a mitigation to unblock code scanning default setup and code quality features on pull requests.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T15:20:46.766Z","updated_at":"2026-04-20T15:20:46.766Z","display_at":"2026-04-20T15:20:46.766Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vxj0gds24xw2","status":"investigating","body":"We are currently deploying mitigations that should unblock code scanning default setup and code quality features on pull requests.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T14:38:51.903Z","updated_at":"2026-04-20T14:38:51.903Z","display_at":"2026-04-20T14:38:51.903Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"5z07zwvk2wf4","status":"investigating","body":"We are actively working to mitigate an issue affecting code scanning default setup and code quality features on pull requests. Users may experience pull request code scanning and code quality analyses not being triggered on new pull requests. Our engineering team has identified the root cause and working on mitigating the issue.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T13:57:43.196Z","updated_at":"2026-04-20T13:57:43.196Z","display_at":"2026-04-20T13:57:43.196Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"7523x859r9yc","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"m7pgzw1wlfq7","created_at":"2026-04-20T13:28:48.613Z","updated_at":"2026-04-20T13:28:48.613Z","display_at":"2026-04-20T13:28:48.613Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"kcf0thb1zp30","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-17T14:56:22.556Z","updated_at":"2026-04-21T14:20:24.839Z","monitoring_at":"2026-04-17T15:18:05.958Z","resolved_at":"2026-04-17T15:18:14.767Z","impact":"minor","shortlink":"https://stspg.io/96xp38b8l8rn","started_at":"2026-04-17T14:56:22.547Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"3q9yb4wmhct7","status":"resolved","body":"On April 17, 2026, between 14:46 UTC and 15:12 UTC, users experienced a degraded web experience on GitHub.com. During this time, approximately 1.5% of web requests resulted in errors, with some users encountering slow page loads or failed requests. The issue was caused by capacity saturation of a caching component in one of our data center regions. We mitigated the issue by redirecting traffic to an unaffected region and rolling back a recent deployment. The incident was fully resolved at 15:18 UTC. We are taking steps to provide appropriate capacity for this caching path to prevent recurrence.","incident_id":"kcf0thb1zp30","created_at":"2026-04-17T15:18:14.767Z","updated_at":"2026-04-21T14:20:24.835Z","display_at":"2026-04-17T15:18:14.767Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"lmp5n3sc94l2","status":"monitoring","body":"The degradation affecting Issues has been mitigated. We are monitoring to ensure stability.","incident_id":"kcf0thb1zp30","created_at":"2026-04-17T15:18:05.958Z","updated_at":"2026-04-17T15:18:05.958Z","display_at":"2026-04-17T15:18:05.958Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hg2ddc149qsl","status":"investigating","body":"We have isolated a problematic component in our infrastructure and are working to mitigate. We will continue to post updates as we work toward resolution.","incident_id":"kcf0thb1zp30","created_at":"2026-04-17T15:08:06.465Z","updated_at":"2026-04-17T15:08:06.465Z","display_at":"2026-04-17T15:08:06.465Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"z72707j6kl9f","status":"investigating","body":"We are experiencing an issue that impacts approximately 10% of traffic to the web, resulting in slow and failed calls. We are investigating and will continue to post updates as we work toward mitigation.","incident_id":"kcf0thb1zp30","created_at":"2026-04-17T14:57:34.209Z","updated_at":"2026-04-17T14:57:34.209Z","display_at":"2026-04-17T14:57:34.209Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"degraded_performance","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"z0fsqk81n9gc","status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","incident_id":"kcf0thb1zp30","created_at":"2026-04-17T14:56:58.060Z","updated_at":"2026-04-17T14:56:58.060Z","display_at":"2026-04-17T14:56:58.060Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"degraded_performance"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"25lkr63xtht8","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"kcf0thb1zp30","created_at":"2026-04-17T14:56:22.609Z","updated_at":"2026-04-17T14:56:22.609Z","display_at":"2026-04-17T14:56:22.609Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"kr09ddfgbfsf","name":"Issues","status":"operational","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2026-04-27T21:33:20.375Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"bjyfg0yz1lwj","name":"Incident with Codespaces","status":"resolved","created_at":"2026-04-16T15:06:56.493Z","updated_at":"2026-04-21T15:52:29.813Z","monitoring_at":"2026-04-16T18:22:24.609Z","resolved_at":"2026-04-16T18:28:07.238Z","impact":"major","shortlink":"https://stspg.io/1pw46p8cqh5s","started_at":"2026-04-16T15:06:56.487Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"rq6mwnq451dk","status":"resolved","body":"On April 16, 2026 between 09:30 UTC and 17:15 UTC, users experienced failures when attempting to connect to GitHub Codespaces via the VS Code editor. During this time, approximately 40% of codespace start operations failed. Users connecting via SSH were not impacted.  \u003cbr /\u003e\u003cbr /\u003eThe issue was caused by a failure in an upstream download service that prevented the VS Code Server from being retrieved during codespace startup.  The impact was mitigated by implementing a workaround to use an alternative download path when the primary endpoint is degraded.  \u003cbr /\u003e\u003cbr /\u003eWe are working with the upstream dependency to address the root cause of the download service failure, and we are improving our fallback mechanisms to reduce the impact of similar upstream failures in the future.","incident_id":"bjyfg0yz1lwj","created_at":"2026-04-16T18:28:07.238Z","updated_at":"2026-04-21T15:52:29.808Z","display_at":"2026-04-16T18:28:07.238Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"h9lcwht0spj3","status":"monitoring","body":"The degradation affecting Codespaces has been mitigated. We are monitoring to ensure stability.","incident_id":"bjyfg0yz1lwj","created_at":"2026-04-16T18:22:24.609Z","updated_at":"2026-04-16T18:22:24.609Z","display_at":"2026-04-16T18:22:24.609Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"major_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"sqdsnxfyt2z8","status":"investigating","body":"Our provider is implementing a mitigation and we are seeing signs of recovery.","incident_id":"bjyfg0yz1lwj","created_at":"2026-04-16T16:37:28.292Z","updated_at":"2026-04-16T16:37:28.292Z","display_at":"2026-04-16T16:37:28.292Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"major_outage","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"tt5t0myhzhd5","status":"investigating","body":"We found an issue that impacts 70% of Codespaces. We are engaged with the provider and working towards mitigation.","incident_id":"bjyfg0yz1lwj","created_at":"2026-04-16T15:49:13.291Z","updated_at":"2026-04-16T15:49:13.291Z","display_at":"2026-04-16T15:49:13.291Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"major_outage","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"g5y78yckkw36","status":"investigating","body":"Codespaces is experiencing degraded availability. We are continuing to investigate.","incident_id":"bjyfg0yz1lwj","created_at":"2026-04-16T15:41:45.122Z","updated_at":"2026-04-16T15:41:45.122Z","display_at":"2026-04-16T15:41:45.122Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"cbvg07h1y3m6","status":"investigating","body":"We are experiencing degraded performance in Codespaces related to creating a new Codespace or starting an existing Codespace from the VS Code editor. SSH connections to Codespaces are not impacted. We are working toward mitigation and will continue to keep you updated on progress.","incident_id":"bjyfg0yz1lwj","created_at":"2026-04-16T15:08:26.875Z","updated_at":"2026-04-16T15:08:26.875Z","display_at":"2026-04-16T15:08:26.875Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"3f4wphf8n9rg","status":"investigating","body":"We are investigating reports of degraded performance for Codespaces","incident_id":"bjyfg0yz1lwj","created_at":"2026-04-16T15:06:56.555Z","updated_at":"2026-04-16T15:06:56.555Z","display_at":"2026-04-16T15:06:56.555Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"h2ftsgbw7kmk","name":"Codespaces","status":"operational","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2026-04-23T15:02:29.440Z","position":11,"description":"Orchestration and Compute for GitHub Codespaces","showcase":true,"start_date":"2021-08-11","group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"gldshbt8tvdg","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-14T01:57:55.842Z","updated_at":"2026-04-17T16:57:17.634Z","monitoring_at":"2026-04-14T06:07:07.746Z","resolved_at":"2026-04-14T06:08:16.750Z","impact":"minor","shortlink":"https://stspg.io/gnnrv0yd0d3m","started_at":"2026-04-14T01:57:55.834Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"q6nmbs13rc7g","status":"resolved","body":"On April 14, between 00:58 UTC and 06:08 UTC, GitHub Enterprise Cloud customers experienced 500 errors when attempting to access Copilot Insights pages which was caused by an authentication failure in our metrics pipeline. We fully mitigated the issue and validated the fix in production. Approximately 709 users were impacted. The total impact duration was approximately 5 hours and 10 minutes. \u003cbr /\u003e\u003cbr /\u003eOur investigation determined the incident was caused by a change in a tenant credential which caused authentication errors to retrieve the required data needed on our Copilot Insights pages. \u003cbr /\u003e\u003cbr /\u003eWe understand this disruption impacted customers' ability to access the Copilot Insights page. To prevent similar issues and reduce resolution time in the future, we are investing in improved diagnostics tooling to quickly identify the root cause of failures, enhanced monitoring, and alerting to detect issues at a more granular level. \u003cbr /\u003e\u003cbr /\u003eGitHub is a critical infrastructure for your work, your teams, and your businesses. We are focused on these remediations and continued reliability improvements for Copilot Insights and related metrics experiences.","incident_id":"gldshbt8tvdg","created_at":"2026-04-14T06:08:16.750Z","updated_at":"2026-04-17T16:57:17.628Z","display_at":"2026-04-14T06:08:16.750Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"j81bfbh3hsqd","status":"monitoring","body":"This incident has been resolved. We will continue to monitor to ensure stability. Thank you for your patience and understanding as we addressed this issue.","incident_id":"gldshbt8tvdg","created_at":"2026-04-14T06:07:55.935Z","updated_at":"2026-04-14T06:07:55.935Z","display_at":"2026-04-14T06:07:55.935Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"630wc0hbvz8r","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"gldshbt8tvdg","created_at":"2026-04-14T06:07:07.746Z","updated_at":"2026-04-14T06:07:07.746Z","display_at":"2026-04-14T06:07:07.746Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hsppgpy4p1wq","status":"investigating","body":"We identified an issue that impacts the Copilot Dashboard on the Insights tab and are working on mitigation. We will continue to keep you updated on progress.","incident_id":"gldshbt8tvdg","created_at":"2026-04-14T04:40:12.226Z","updated_at":"2026-04-14T04:40:12.226Z","display_at":"2026-04-14T04:40:12.226Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vlj9924cbkrt","status":"investigating","body":"The team continues to investigate issues accessing with Copilot Dashboard on the Insights tab.  We will continue providing updates on the progress towards mitigation.","incident_id":"gldshbt8tvdg","created_at":"2026-04-14T03:47:20.649Z","updated_at":"2026-04-14T03:47:20.649Z","display_at":"2026-04-14T03:47:20.649Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"gvhkr6m4swrz","status":"investigating","body":"The Copilot Dashboard on the Insights tab is not accessible and we are continuing to investigate.","incident_id":"gldshbt8tvdg","created_at":"2026-04-14T02:40:21.315Z","updated_at":"2026-04-14T02:40:21.315Z","display_at":"2026-04-14T02:40:21.315Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hfzfp82vzt3l","status":"investigating","body":"Degradation of Service - Insights Page","incident_id":"gldshbt8tvdg","created_at":"2026-04-14T02:37:51.420Z","updated_at":"2026-04-14T02:37:51.420Z","display_at":"2026-04-14T02:37:51.420Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"0wm1c3k99z25","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"gldshbt8tvdg","created_at":"2026-04-14T01:57:55.882Z","updated_at":"2026-04-14T01:57:55.882Z","display_at":"2026-04-14T01:57:55.882Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"96r9hyz795sv","name":"Incident with Pages","status":"resolved","created_at":"2026-04-13T19:56:04.146Z","updated_at":"2026-04-16T23:53:42.333Z","monitoring_at":"2026-04-13T20:30:10.950Z","resolved_at":"2026-04-13T20:35:37.441Z","impact":"major","shortlink":"https://stspg.io/4fgzqt720l93","started_at":"2026-04-13T19:56:04.127Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"1lnfr6fsxvdx","status":"resolved","body":"On Sunday April 13th, 2026, between 18:53 UTC and 20:30 UTC, the GitHub Pages service experienced elevated error rates. On average, the error rate was 10.58% and peaked at 12.77% of requests to the service, resulting in approximately 17.5 million failed requests returning HTTP 500 errors. This was due to an automated DNS management tool (octodns) erroneously deleting a DNS record for a Pages backend storage host after its upstream data source intermittently failed to return the record, causing the tool to treat it as stale and remove it.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by re-creating the deleted DNS record. To prevent future incidents, we are implementing availability-zone-tolerant routing in the Pages frontend so that an unresolvable backend host triggers failover to healthy hosts rather than returning errors, adding safeguards to prevent automated deletion of DNS records owned by other systems, and improving logging and alerting for DNS resolution failures in the Pages serving path.","incident_id":"96r9hyz795sv","created_at":"2026-04-13T20:35:37.441Z","updated_at":"2026-04-16T23:53:42.328Z","display_at":"2026-04-13T20:35:37.441Z","affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"3my2fzv52vtw","status":"monitoring","body":"We have mitigated the issue with Pages.","incident_id":"96r9hyz795sv","created_at":"2026-04-13T20:32:11.107Z","updated_at":"2026-04-13T20:32:11.107Z","display_at":"2026-04-13T20:32:11.107Z","affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hvdkpvrvhgh5","status":"monitoring","body":"The degradation affecting Pages has been mitigated. We are monitoring to ensure stability.","incident_id":"96r9hyz795sv","created_at":"2026-04-13T20:30:10.950Z","updated_at":"2026-04-13T20:30:10.950Z","display_at":"2026-04-13T20:30:10.950Z","affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"ck5n37w8jg9v","status":"investigating","body":"We are investigating reports of issues with Pages. We will continue to keep users updated on progress towards mitigation.","incident_id":"96r9hyz795sv","created_at":"2026-04-13T19:57:17.687Z","updated_at":"2026-04-13T19:57:17.687Z","display_at":"2026-04-13T19:57:17.687Z","affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"major_outage","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"xj3z023kx144","status":"investigating","body":"We are investigating reports of degraded availability for Pages","incident_id":"96r9hyz795sv","created_at":"2026-04-13T19:56:04.215Z","updated_at":"2026-04-13T19:56:04.215Z","display_at":"2026-04-13T19:56:04.215Z","affected_components":[{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"major_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"vg70hn9s2tyj","name":"Pages","status":"operational","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2026-04-13T20:30:10.916Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"1ghdqdbyyjmt","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-13T16:41:27.425Z","updated_at":"2026-04-17T01:48:32.703Z","monitoring_at":null,"resolved_at":"2026-04-13T17:40:09.946Z","impact":"minor","shortlink":"https://stspg.io/qg4svxjjz3hm","started_at":"2026-04-13T16:41:27.419Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"yy9csnbnd1x4","status":"resolved","body":"On April 13, 2026, between 14:41 UTC and 17:29 UTC, the Copilot service experienced degraded performance. All Copilot users were impacted by increased latency, and approximately 20% experienced request failures when interacting with Copilot Cloud Agent (CCA). On average, request latency increased to approximately 950ms. The GitHub User Dashboard also displayed intermittent errors loading Copilot quota information. CCA and the User Dashboard were impacted for approximately 2 hours and 56 minutes. \u003cbr /\u003e\u003cbr /\u003eThis was due to an infrastructure change that reduced the available compute capacity for a backend service responsible for Copilot rate limiting and quota management. The reduced capacity caused resource exhaustion under normal traffic load, leading to cascading failures in downstream request processing. \u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by increasing compute resources allocated to the affected service and scaling out the number of service instances to distribute load more effectively. \u003cbr /\u003e\u003cbr /\u003eWe are working to improve proactive capacity monitoring to detect resource degradation before it impacts users, reviewing retry and timeout configurations across dependent services to reduce amplification during degraded states, and evaluating connection management strategies to improve resilience under constrained resources.","incident_id":"1ghdqdbyyjmt","created_at":"2026-04-13T17:40:09.946Z","updated_at":"2026-04-17T01:48:32.698Z","display_at":"2026-04-13T17:40:09.946Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"p42l2w94rnw7","status":"investigating","body":"We have identified the root cause and are rolling out a fix for Copilot. The services should now be in recovery, with expected full recovery in 5 to 10 minutes.","incident_id":"1ghdqdbyyjmt","created_at":"2026-04-13T16:59:20.309Z","updated_at":"2026-04-13T16:59:20.309Z","display_at":"2026-04-13T16:59:20.309Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"3zzfz0n15vpm","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"1ghdqdbyyjmt","created_at":"2026-04-13T16:41:27.467Z","updated_at":"2026-04-13T16:41:27.467Z","display_at":"2026-04-13T16:41:27.467Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"8hn4vclxz1lq","name":"Problems with third-party Claude and Codex Agent sessions not being listed in the agents tab dashboard","status":"resolved","created_at":"2026-04-10T13:07:21.793Z","updated_at":"2026-04-14T17:56:47.029Z","monitoring_at":null,"resolved_at":"2026-04-10T13:28:23.098Z","impact":"minor","shortlink":"https://stspg.io/3lyzg9tk7fl8","started_at":"2026-04-10T13:07:21.787Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"cbwxnq5s63v1","status":"resolved","body":"On April 9, 2026, between 22:59 UTC and April 10, 2026, 13:24 UTC, the Copilot Mission Control service was degraded and did not display Claude and Codex Cloud Agent sessions in the agents tab dashboard. Customers were unable to see, list, or manage their third party agent sessions during this period. The underlying agent sessions continued to function normally. This was a visibility and management issue only, and no HTTP errors were generated. The API returned successful responses with incomplete results, with an average error rate of 0% and a maximum error rate of 0%. This was due to a code change that introduced a filter which inadvertently excluded third party agent sessions.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by reverting the problematic code change and deploying the fix to production.\u003cbr /\u003e\u003cbr /\u003eWe are working to add automated monitoring for dashboard content visibility and improve integration test coverage for third party agent session listing to reduce our time to detection and mitigation of issues like this one in the future.","incident_id":"8hn4vclxz1lq","created_at":"2026-04-10T13:28:23.098Z","updated_at":"2026-04-14T17:56:47.023Z","display_at":"2026-04-10T13:28:23.098Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"g2qvl9r0s6p1","status":"investigating","body":"We are investigating third party Claude and Codex Cloud Agent sessions not being listed in the agents tab dashboard.","incident_id":"8hn4vclxz1lq","created_at":"2026-04-10T13:08:00.892Z","updated_at":"2026-04-10T13:08:00.892Z","display_at":"2026-04-10T13:08:00.892Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hy1bfp50kswg","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"8hn4vclxz1lq","created_at":"2026-04-10T13:07:21.824Z","updated_at":"2026-04-10T13:07:21.824Z","display_at":"2026-04-10T13:07:21.824Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"2rqwxl8y7m0j","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-09T16:20:03.691Z","updated_at":"2026-04-16T13:17:39.023Z","monitoring_at":null,"resolved_at":"2026-04-09T20:36:52.987Z","impact":"major","shortlink":"https://stspg.io/h8kh83zv9jyr","started_at":"2026-04-09T16:20:03.684Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"432085xhzks1","status":"resolved","body":"On April 9, 2026, between 16:05 UTC and 20:36 UTC, the Copilot cloud agent service was degraded, causing new agent sessions to be delayed or fail to start. Users who attempted to start Copilot cloud agent sessions during this period experienced jobs getting stuck in the queue, with wait times peaking at 54 minutes compared to the normal 15–40 seconds. On average, approximately 84% of requests to start agent sessions failed, peaking at 97.5% during the worst period.\u003cbr /\u003e\u003cbr /\u003eThis was due to an internal service exceeding API rate limits, compounded by a caching bug that persisted the rate-limited state beyond the actual rate limit window, causing recurring outage waves rather than a single recovery.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by deploying a configuration change to bypass the affected cache and shifting API traffic to an alternative authentication path that reduced rate limit exposure. We have since added automated monitoring and alerting for this failure mode, deployed per-endpoint rate limit controls, and added caching for high-traffic API calls to reduce overall load. We are also working on longer-term improvements to rate limit isolation and traffic management to prevent similar issues in the future.\u003cbr /\u003e\u003cbr /\u003eThis incident shared the same underlying root causes with an incident declared in the time frame https://www.githubstatus.com/incidents/zn1t56bfxdzg","incident_id":"2rqwxl8y7m0j","created_at":"2026-04-09T20:36:52.987Z","updated_at":"2026-04-16T13:17:39.018Z","display_at":"2026-04-09T20:36:52.987Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"6bnsmx3kw9ws","status":"investigating","body":"We continue to investigate periodic delays in Copilot Cloud Agent job processing","incident_id":"2rqwxl8y7m0j","created_at":"2026-04-09T19:52:45.027Z","updated_at":"2026-04-09T19:52:45.027Z","display_at":"2026-04-09T19:52:45.027Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hv0g80y2gzsb","status":"investigating","body":"We are continuing to investigate Copilot Cloud Agent job delays","incident_id":"2rqwxl8y7m0j","created_at":"2026-04-09T18:57:54.024Z","updated_at":"2026-04-09T18:57:54.024Z","display_at":"2026-04-09T18:57:54.024Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"kjm69bxg6xy4","status":"investigating","body":"Copilot Cloud Agent jobs are being processed and we are monitoring recovery","incident_id":"2rqwxl8y7m0j","created_at":"2026-04-09T17:48:11.684Z","updated_at":"2026-04-09T17:48:11.684Z","display_at":"2026-04-09T17:48:11.684Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"23g1jhrcmbk0","status":"investigating","body":"We are investigating delays processing Copilot Cloud Agent jobs","incident_id":"2rqwxl8y7m0j","created_at":"2026-04-09T16:57:17.884Z","updated_at":"2026-04-09T16:57:17.884Z","display_at":"2026-04-09T16:57:17.884Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"9r58tgq0k0jz","status":"investigating","body":"We are experiencing issues where jobs are being delayed to start for copilot coding agent","incident_id":"2rqwxl8y7m0j","created_at":"2026-04-09T16:20:12.473Z","updated_at":"2026-04-09T16:20:12.473Z","display_at":"2026-04-09T16:20:12.473Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"zp5fmj97t32r","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"2rqwxl8y7m0j","created_at":"2026-04-09T16:20:03.750Z","updated_at":"2026-04-09T16:20:03.750Z","display_at":"2026-04-09T16:20:03.750Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"zn1t56bfxdzg","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-09T09:50:21.547Z","updated_at":"2026-04-16T13:20:53.985Z","monitoring_at":"2026-04-09T10:15:35.976Z","resolved_at":"2026-04-09T10:15:37.484Z","impact":"major","shortlink":"https://stspg.io/v15brqnhy95t","started_at":"2026-04-09T09:50:21.538Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"l2ffqynv742l","status":"resolved","body":"On April 9, 2026, between 09:05 UTC and 19:05 UTC, the Copilot coding agent service was degraded and users experienced significant delays starting new agent sessions. Approximately 84% of new agent session requests were delayed across four separate outage waves, with queue wait times peaking at 54 minutes compared to a normal baseline of 15–40 seconds. On average, the error rate was 83.9% and peaked at 97.5% of requests to the service. Approximately 22,700 workflow creations were delayed or failed during the incident.\u003cbr /\u003e\u003cbr /\u003eThis was due to a bug in our rate limiting logic that incorrectly applied a rate limit globally across all users, rather than scoping it to the individual installation that triggered the limit. A contributing factor was a surge in API traffic from a client update that increased requests to an internal endpoint by 3–4x, which accelerated rate limit exhaustion.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by disabling the faulty rate limit caching mechanism via feature flag and updating our service to use per-installation credentials for API calls, ensuring rate limits are correctly scoped to individual installations.\u003cbr /\u003e\u003cbr /\u003eWe have since added automated monitoring and alerting to detect this failure mode proactively, deployed fixes to reduce unnecessary API traffic through caching improvements, and are continuing work to further isolate rate limit scoping across client types to prevent similar issues in the future.\u003cbr /\u003e\u003cbr /\u003eThis incident shared the same underlying root causes with an incident declared in the time frame https://www.githubstatus.com/incidents/2rqwxl8y7m0j","incident_id":"zn1t56bfxdzg","created_at":"2026-04-09T10:15:37.484Z","updated_at":"2026-04-16T13:20:53.980Z","display_at":"2026-04-09T10:15:37.484Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"fx2jmnp1kb70","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"zn1t56bfxdzg","created_at":"2026-04-09T10:15:35.976Z","updated_at":"2026-04-09T10:15:35.976Z","display_at":"2026-04-09T10:15:35.976Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"xwl1920l89q9","status":"investigating","body":"We are investigating an issue affecting GitHub Copilot coding agent. Users may experience significant delays when starting new agent sessions, with jobs remaining queued longer than expected. Our team has identified increased load as a contributing factor and is actively working to restore normal performance.","incident_id":"zn1t56bfxdzg","created_at":"2026-04-09T09:57:01.165Z","updated_at":"2026-04-09T09:57:01.165Z","display_at":"2026-04-09T09:57:01.165Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"spqwghgtqdp5","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"zn1t56bfxdzg","created_at":"2026-04-09T09:50:21.595Z","updated_at":"2026-04-09T09:50:21.595Z","display_at":"2026-04-09T09:50:21.595Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"bnvg7qml7krl","name":"Disruption with GitHub notifications","status":"resolved","created_at":"2026-04-09T04:42:06.164Z","updated_at":"2026-04-15T18:52:20.393Z","monitoring_at":"2026-04-09T04:57:04.537Z","resolved_at":"2026-04-09T04:57:17.755Z","impact":"minor","shortlink":"https://stspg.io/jndfpdy29y94","started_at":"2026-04-09T04:42:06.156Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"zwdb0khh9t1c","status":"resolved","body":"On April 9, 2026, between 03:22 UTC and 04:49 UTC, GitHub Notifications experienced degraded availability. During this time, approximately 45% of requests to the notifications service returned errors, with a peak error rate of approximately 54%, preventing affected users from successfully viewing or interacting with their  notifications service. The issue was identified and resolved, restoring the service to full availability.\u003cbr /\u003e\u003cbr /\u003eWe are working to improve our metrics to reduce time to detection and mitigation for similar issues in the future.","incident_id":"bnvg7qml7krl","created_at":"2026-04-09T04:57:17.755Z","updated_at":"2026-04-15T18:52:20.388Z","display_at":"2026-04-09T04:57:17.755Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"t93j44cgl242","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"bnvg7qml7krl","created_at":"2026-04-09T04:57:04.537Z","updated_at":"2026-04-09T04:57:04.537Z","display_at":"2026-04-09T04:57:04.537Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"2tjpmxwwbbhj","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"bnvg7qml7krl","created_at":"2026-04-09T04:42:06.203Z","updated_at":"2026-04-09T04:42:06.203Z","display_at":"2026-04-09T04:42:06.203Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"d96l71t3h63k","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-04-02T17:49:36.922Z","updated_at":"2026-04-06T17:17:13.732Z","monitoring_at":"2026-04-02T21:48:42.038Z","resolved_at":"2026-04-02T21:48:43.705Z","impact":"minor","shortlink":"https://stspg.io/5b7098zw5zw7","started_at":"2026-04-02T17:49:36.914Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"p30qmt2ph1v2","status":"resolved","body":"Between 15:20 and 20:18 UTC on Thursday April 2, Copilot Cloud Agent entered a period of reduced performance.  Due to an internal feature being developed for Copilot Code Review, the Copilot Cloud Agent infrastructure started to receive an increased number of jobs.  This load eventually caused us to hit an internal rate limit, causing all work to suspend for an hour.  During this hour, some new jobs would time out, while others would resume once rate limiting ended.  Roughly 40% of jobs in this period were affected.\u003cbr /\u003e\u003cbr /\u003eOnce the cause of this rate limiting was identified, we were able to disable the new CCR feature via a feature flag.  Once the jobs that were already in the queue were able to clear, we didn't see additional instances of rate limiting afterwards.","incident_id":"d96l71t3h63k","created_at":"2026-04-02T21:48:43.705Z","updated_at":"2026-04-06T17:17:13.727Z","display_at":"2026-04-02T21:48:43.705Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"jj8hxy863n2q","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"d96l71t3h63k","created_at":"2026-04-02T21:48:42.038Z","updated_at":"2026-04-02T21:48:42.038Z","display_at":"2026-04-02T21:48:42.038Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"gyzj3nyb3985","status":"investigating","body":"Although we are observing recovery once again, we expect continued periods of degradation. \u003cbr /\u003e\u003cbr /\u003eWork that is queued during times of degradation does eventually get processed. \u003cbr /\u003e\u003cbr /\u003eWe continue to investigate and find a mitigation, and will update again within 2 hours.","incident_id":"d96l71t3h63k","created_at":"2026-04-02T20:35:53.857Z","updated_at":"2026-04-02T20:35:53.857Z","display_at":"2026-04-02T20:35:53.857Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"666k93jk54t4","status":"investigating","body":"This issue has recurred. Customers will once again experience false job starts when assigning tasks to Copilot Cloud Agent. \u003cbr /\u003e\u003cbr /\u003eWe are still investigating and trying to understand the pattern of degradation.","incident_id":"d96l71t3h63k","created_at":"2026-04-02T19:28:43.919Z","updated_at":"2026-04-02T19:28:43.919Z","display_at":"2026-04-02T19:28:43.919Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"2fl1p3hqzyxp","status":"investigating","body":"We are once again seeing recovery with Copilot Cloud Agent job starts. \u003cbr /\u003e\u003cbr /\u003eWe are keeping this open while we verify this won't recur.","incident_id":"d96l71t3h63k","created_at":"2026-04-02T18:25:50.677Z","updated_at":"2026-04-02T18:25:50.677Z","display_at":"2026-04-02T18:25:50.677Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"v6z2j3ckwfvz","status":"investigating","body":"When assigning tasks to Copilot Cloud Agent, the task will appear to be working, but may not actually be running.\u003cbr /\u003e\u003cbr /\u003eWe are investigating.","incident_id":"d96l71t3h63k","created_at":"2026-04-02T17:59:23.184Z","updated_at":"2026-04-02T17:59:23.184Z","display_at":"2026-04-02T17:59:23.184Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"k7p8k13kp8j1","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"d96l71t3h63k","created_at":"2026-04-02T17:49:36.979Z","updated_at":"2026-04-02T17:49:36.979Z","display_at":"2026-04-02T17:49:36.979Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"j3sgbdw2lw3c","name":"Copilot Coding Agent failing to start some jobs","status":"resolved","created_at":"2026-04-02T16:18:04.776Z","updated_at":"2026-04-07T19:09:23.970Z","monitoring_at":null,"resolved_at":"2026-04-02T16:30:05.082Z","impact":"minor","shortlink":"https://stspg.io/qrn1lx3vp2cn","started_at":"2026-04-02T16:18:04.766Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"bnbf5pq5736z","status":"resolved","body":"Between 15:20 and 20:18 UTC on Thursday April 2, Copilot Cloud Agent entered a period of reduced performance. Due to an internal feature being developed for Copilot Code Review, the Copilot Cloud Agent infrastructure started to receive an increased number of jobs. This load eventually caused us to hit an internal rate limit, causing all work to suspend for an hour. During this hour, some new jobs would time out, while others would resume once rate limiting ended. Roughly 40% of jobs in this period were affected.\u003cbr /\u003e\u003cbr /\u003eOnce the cause of this rate limiting was identified, we were able to disable the new CCR feature via a feature flag. Once the jobs that were already in the queue were able to clear, we didn't see additional instances of rate limiting afterwards.\u003cbr /\u003e\u003cbr /\u003eThis was the same incident declared in https://www.githubstatus.com/incidents/d96l71t3h63k","incident_id":"j3sgbdw2lw3c","created_at":"2026-04-02T16:30:05.082Z","updated_at":"2026-04-07T19:09:23.963Z","display_at":"2026-04-02T16:30:05.082Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"dx9kbq2nhk3v","status":"investigating","body":"When assigning tasks to Copilot Cloud Agent, the task will appear to be working, but may not actually be running. \u003cbr /\u003e\u003cbr /\u003eWe are investigating.","incident_id":"j3sgbdw2lw3c","created_at":"2026-04-02T16:28:28.533Z","updated_at":"2026-04-02T16:28:28.533Z","display_at":"2026-04-02T16:28:28.533Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"l1bmwrdwtxp4","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"j3sgbdw2lw3c","created_at":"2026-04-02T16:18:04.847Z","updated_at":"2026-04-02T16:18:04.847Z","display_at":"2026-04-02T16:18:04.847Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"xs2p9z3b73r3","name":"Disruption with GitHub's code search","status":"resolved","created_at":"2026-04-01T15:02:54.256Z","updated_at":"2026-04-10T19:02:47.932Z","monitoring_at":null,"resolved_at":"2026-04-01T23:45:45.461Z","impact":"major","shortlink":"https://stspg.io/1zgx17jdj4k7","started_at":"2026-04-01T15:02:54.247Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"zg5m3gqq5h6g","status":"resolved","body":"On April 1st, 2026 between 14:40 and 17:00 UTC the GitHub code search service had an outage which resulted in users being unable to perform searches.\u003cbr /\u003e\u003cbr /\u003eThe issue was initially caused by an upgrade to the code search Kafka cluster ZooKeeper instances which caused a loss of quorum.  This resulted in application-level data inconsistencies which required the index to be reset to a point in time before the loss of quorum occurred. Meanwhile, an accidental deploy resulted in query services losing their shard-to-host mappings, which are typically propagated by Kafka.\u003cbr /\u003e\u003cbr /\u003eWe remediated the problem by performing rolling restarts in the Kafka cluster, allowing quorum to be reestablished. From there we were able to reset our index to a point in time before the inconsistencies occurred.\u003cbr /\u003e\u003cbr /\u003eThe team is working on ways to improve our time to respond and mitigate issues relating to Kafka in the future.","incident_id":"xs2p9z3b73r3","created_at":"2026-04-01T23:45:45.461Z","updated_at":"2026-04-10T19:02:47.926Z","display_at":"2026-04-01T23:45:45.461Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"tjfsjvnry8vy","status":"investigating","body":"Code search has recovered and is serving production traffic.","incident_id":"xs2p9z3b73r3","created_at":"2026-04-01T23:45:27.356Z","updated_at":"2026-04-01T23:45:27.356Z","display_at":"2026-04-01T23:45:27.356Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"bttjlfx0nppj","status":"investigating","body":"We have stabilized Code Search infrastructure, and are in the final stages of validation before slowly reintroducing production traffic.","incident_id":"xs2p9z3b73r3","created_at":"2026-04-01T22:00:46.313Z","updated_at":"2026-04-01T22:00:46.313Z","display_at":"2026-04-01T22:00:46.313Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"rg9gs05n3rvj","status":"investigating","body":"We are still working on recovering back to a serviceable state and expect to have a more substantial update within another two hours.","incident_id":"xs2p9z3b73r3","created_at":"2026-04-01T19:37:57.575Z","updated_at":"2026-04-01T19:37:57.575Z","display_at":"2026-04-01T19:37:57.575Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"sq7z8g7nfjcm","status":"investigating","body":"We are observing some recovery for Code Search queries, but customers should be aware that the data being served may be stale, especially for changes that took place after 07:00 UTC today (1 April 2026). We are still working on recovering our ingestion pipeline, and synchronizing the indexed data.\u003cbr /\u003e\u003cbr /\u003eWe will update again within 2 hours.","incident_id":"xs2p9z3b73r3","created_at":"2026-04-01T17:48:15.072Z","updated_at":"2026-04-01T17:48:15.072Z","display_at":"2026-04-01T17:48:15.072Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"cqmtd386yjvc","status":"investigating","body":"We identified an issue in our ingestion pipeline that degraded the freshness of Code Search results. While fixing the issue with the ingestion pipeline, a deployment caused a loss of dynamic configuration which is causing most requests for Code Search results to fail. We are working to restore the service and to re-ingest the misaligned data.","incident_id":"xs2p9z3b73r3","created_at":"2026-04-01T16:00:31.111Z","updated_at":"2026-04-01T16:00:31.111Z","display_at":"2026-04-01T16:00:31.111Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"y3gmjyzzgh8w","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"xs2p9z3b73r3","created_at":"2026-04-01T15:02:54.300Z","updated_at":"2026-04-01T15:02:54.300Z","display_at":"2026-04-01T15:02:54.300Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"00h92ct3ggvv","name":"GitHub audit logs are unavailable","status":"resolved","created_at":"2026-04-01T16:06:11.166Z","updated_at":"2026-04-02T03:17:14.769Z","monitoring_at":null,"resolved_at":"2026-04-01T16:10:17.528Z","impact":"major","shortlink":"https://stspg.io/vs91g30yd70s","started_at":"2026-04-01T16:06:11.159Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"px03r4m92yyw","status":"resolved","body":"On April 1, 2026, between 15:34 UTC and 16:02 UTC, our audit log service lost connectivity to its backing data store due to a failed credential rotation. During this 28-minute window, audit log history was unavailable via both the API and web UI. This resulted in 5xx errors for 4,297 API actors and 127 github.com users. Additionally, events created during this window were delayed by up to 29 minutes in github.com and event streaming. No audit log events were lost; all audit log events were ultimately written and streamed successfully. Customers using GitHub Enterprise Cloud with data residency were not impacted by this incident. \u003cbr /\u003e\u003cbr /\u003eWe were alerted to the infrastructure failure at 15:40 UTC — six minutes after onset — and resolved the issue by recycling the affected environment, restoring full service by 16:02 UTC. We are conducting a thorough review of our credential rotation process to strengthen its resiliency and prevent recurrence. In parallel, we are strengthening our monitoring capabilities to ensure faster detection and earlier visibility into similar issues going forward.","incident_id":"00h92ct3ggvv","created_at":"2026-04-01T16:10:17.528Z","updated_at":"2026-04-02T03:17:14.765Z","display_at":"2026-04-01T16:10:17.528Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"lq78nc8dy330","status":"investigating","body":"A routine credential rotation has failed for our our audit logs service; we have re-deployed our service and are waiting for recovery.","incident_id":"00h92ct3ggvv","created_at":"2026-04-01T16:07:10.857Z","updated_at":"2026-04-01T16:07:10.857Z","display_at":"2026-04-01T16:07:10.857Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"ytkfxwx01r9f","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"00h92ct3ggvv","created_at":"2026-04-01T16:06:11.207Z","updated_at":"2026-04-01T16:06:11.207Z","display_at":"2026-04-01T16:06:11.207Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"tstkmx7pcs5d","name":"Incident with Copilot","status":"resolved","created_at":"2026-04-01T09:58:13.543Z","updated_at":"2026-04-03T19:59:42.012Z","monitoring_at":"2026-04-01T10:56:30.495Z","resolved_at":"2026-04-01T12:41:38.233Z","impact":"minor","shortlink":"https://stspg.io/2xwjn4t1r8jy","started_at":"2026-04-01T09:58:13.534Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"y3swsm63sr56","status":"resolved","body":"On April 1, 2026, between 07:29 and 12:41 UTC, some customers experienced elevated 5xx errors and increased latency when using GitHub Copilot features that rely on `/agents/sessions` endpoints (including creating or viewing agent sessions). The issue was caused by resource exhaustion in one of the Copilot backend services handling these requests, in turn, causing timeouts and failed requests. We mitigated the incident by increasing the service’s available compute resources and tuning its runtime concurrency settings. Service health returned to normal and the incident was fully resolved by 12:41 UTC.","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T12:41:38.233Z","updated_at":"2026-04-03T19:59:42.006Z","display_at":"2026-04-01T12:41:38.233Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"mbb4gc7y8mth","status":"monitoring","body":"The success rate and latency for creating and viewing agent sessions has stabilized at baseline levels, we are continuing to monitor recovery","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T12:10:19.100Z","updated_at":"2026-04-01T12:10:19.100Z","display_at":"2026-04-01T12:10:19.100Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"yh170yb3tvw6","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T12:02:33.973Z","updated_at":"2026-04-01T12:02:33.973Z","display_at":"2026-04-01T12:02:33.973Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"2zcbw97dhw85","status":"monitoring","body":"The success rate for creating and viewing agent sessions has stabilized, and we're continuing to monitor latency, which is trending toward baseline levels.","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T11:37:22.534Z","updated_at":"2026-04-01T11:37:22.534Z","display_at":"2026-04-01T11:37:22.534Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"00rr5c743ktq","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T11:24:35.865Z","updated_at":"2026-04-01T11:24:35.865Z","display_at":"2026-04-01T11:24:35.865Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"kk4nwb6kv3hb","status":"monitoring","body":"The degradation affecting Copilot has been mitigated. We are monitoring to ensure stability.","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T10:56:30.495Z","updated_at":"2026-04-01T10:56:30.495Z","display_at":"2026-04-01T10:56:30.495Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"0f7j3my4tsqf","status":"investigating","body":"Users may see increased latency and intermittent errors when viewing or creating agent sessions. We are working on mitigations to return to baseline performance and success rate.","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T10:31:24.170Z","updated_at":"2026-04-01T10:31:24.170Z","display_at":"2026-04-01T10:31:24.170Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"r9pr21w34p03","status":"investigating","body":"We are investigating reports of issues with service(s): Copilot Dotcom Agents. We will continue to keep users updated on progress towards mitigation.","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T10:00:26.914Z","updated_at":"2026-04-01T10:00:26.914Z","display_at":"2026-04-01T10:00:26.914Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"kyyy6lp1kwsc","status":"investigating","body":"We are investigating reports of degraded performance for Copilot","incident_id":"tstkmx7pcs5d","created_at":"2026-04-01T09:58:13.638Z","updated_at":"2026-04-01T09:58:13.638Z","display_at":"2026-04-01T09:58:13.638Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"pjmpxvq2cmr2","name":"Copilot","status":"operational","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2026-04-23T17:03:00.903Z","position":10,"description":null,"showcase":true,"start_date":"2022-06-21","group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"ml7wplmxbt5l","name":"Incident with Pull Requests: High percentage of 500s","status":"resolved","created_at":"2026-03-31T15:05:39.330Z","updated_at":"2026-04-03T19:37:17.233Z","monitoring_at":"2026-03-31T21:16:01.703Z","resolved_at":"2026-03-31T21:23:43.015Z","impact":"minor","shortlink":"https://stspg.io/lvkqg4h1rntp","started_at":"2026-03-31T15:05:39.322Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"lfv66d6q577b","status":"resolved","body":"On Monday March 31st, 2026, between 13:53 UTC and 21:23 UTC the Pull Requests service experienced elevated latency and failures. On average, the error rate was 0.15% and peaked at 0.28% of requests to the service. This was due to a change in garbage collection (GC) settings for a Go-based internal service that provides access to Git repository data. The changes caused more frequent GC activity and elevated CPU consumption on a subset of storage nodes, increasing latency and failure rates for some internal API operations.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by reverting the GC changes. To prevent future incidents and improve time to detection and mitigation, we are instrumenting additional metrics and alerting for GC-related behavior, improving our visibility into other signals that could cause degraded impact of this type, and updating our best practices and standards for garbage collection in Go-based services.\u003cbr /\u003e","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T21:23:43.015Z","updated_at":"2026-04-03T19:37:17.228Z","display_at":"2026-03-31T21:23:43.015Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"mp23cbtbf1wp","status":"monitoring","body":"The degradation affecting Pull Requests has been mitigated. We are monitoring to ensure stability.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T21:16:01.703Z","updated_at":"2026-03-31T21:16:01.703Z","display_at":"2026-03-31T21:16:01.703Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"sjw7tg95kk25","status":"investigating","body":"We continue to see a small subset of repositories experiencing timeouts and elevated latency in Pull Requests, affecting under 1% of requests.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T21:12:17.179Z","updated_at":"2026-03-31T21:12:17.179Z","display_at":"2026-03-31T21:12:17.179Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"l6j1tjwh19pp","status":"investigating","body":"Error rates remain elevated across multiple pull request endpoints. We are pursuing multiple potential mitigations.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T19:28:29.659Z","updated_at":"2026-03-31T19:28:29.659Z","display_at":"2026-03-31T19:28:29.659Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"xmrkfn1kbggh","status":"investigating","body":"We continue to experience elevated error rates affecting Pull Requests. An earlier fix resolved one component of the issue, but some users may still encounter intermittent timeouts when viewing or interacting with pull requests. Our teams are actively investigating the remaining causes.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T18:42:04.963Z","updated_at":"2026-03-31T18:42:04.963Z","display_at":"2026-03-31T18:42:04.963Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"3gwn0zk0qfxp","status":"investigating","body":"We identified an issue causing increased errors when accessing Pull Requests. The mitigation is being applied across our infrastructure and we will continue to provide updates as the mitigation rolls out.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T17:16:32.971Z","updated_at":"2026-03-31T17:16:32.971Z","display_at":"2026-03-31T17:16:32.971Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"r5wljgn8s1fl","status":"investigating","body":"We are seeing recovery in latency and timeouts of requests related to pull requests, even though 500s are still elevated. While we are continuing to investigate, we are applying a mitigation and expect further recovery after it is applied.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T16:35:22.206Z","updated_at":"2026-03-31T16:35:22.206Z","display_at":"2026-03-31T16:35:22.206Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"ndjz3x8zqcfs","status":"investigating","body":"We are continuing to investigate increased 500 errors affecting GitHub services. You may experience intermittent failures when using Pull Requests and other features. We are actively working to identify and resolve the underlying cause.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T16:15:37.558Z","updated_at":"2026-03-31T16:15:37.558Z","display_at":"2026-03-31T16:15:37.558Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"clw0hk60c7xq","status":"investigating","body":"We are investigating increased 500 errors affecting GitHub services. You may experience intermittent failures when using Pull Requests and other features. We are actively working to identify and resolve the underlying cause.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T15:39:48.631Z","updated_at":"2026-03-31T15:39:48.631Z","display_at":"2026-03-31T15:39:48.631Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"rdd0q45c16sc","status":"investigating","body":"We are seeing a higher than average number of 500s due to timeouts across GitHub services. We have a potential mitigation in flight and are continuing to investigate.","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T15:06:48.012Z","updated_at":"2026-03-31T15:06:48.012Z","display_at":"2026-03-31T15:06:48.012Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"zmyz2f63rm5l","status":"investigating","body":"We are investigating reports of degraded performance for Pull Requests","incident_id":"ml7wplmxbt5l","created_at":"2026-03-31T15:05:39.448Z","updated_at":"2026-03-31T15:05:39.448Z","display_at":"2026-03-31T15:05:39.448Z","affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"hhtssxt0f5v2","name":"Pull Requests","status":"degraded_performance","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2026-04-28T15:58:26.154Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"2yb2tvwny50l","name":"Issues with metered billing report generation","status":"resolved","created_at":"2026-03-31T13:47:46.091Z","updated_at":"2026-04-03T01:49:10.571Z","monitoring_at":"2026-03-31T15:01:43.931Z","resolved_at":"2026-03-31T15:10:22.152Z","impact":"minor","shortlink":"https://stspg.io/4pyzfm2phb3b","started_at":"2026-03-31T13:47:46.083Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"k4p6jgypqlcl","status":"resolved","body":"On March 31, 2026, between 06:15 UTC and 15:30 UTC, the GitHub billing usage reports feature was degraded due to reduced server capacity.  Customers requesting billing usage reports and loading the top usage by organization and repository on the billing overview and usage pages were impacted. The average error rate for usage report requests was 15%, peaking at 98% over an eight-minute window. For the billing pages, an average of 56% of requests failed to load the top usage cards. The root cause was an increase in billing usage report requests with large datasets, which exhausted the capacity of the nodes responsible for reporting data. There was no impact on billing charges. \u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by adjusting our auto-scaling thresholds to better meet our capacity needs. We are working to improve our metrics to reduce time to detection and mitigation for similar issues in the future.","incident_id":"2yb2tvwny50l","created_at":"2026-03-31T15:10:22.152Z","updated_at":"2026-04-03T01:49:10.565Z","display_at":"2026-03-31T15:10:22.152Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"y1q1ymgy8bz1","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"2yb2tvwny50l","created_at":"2026-03-31T15:01:43.931Z","updated_at":"2026-03-31T15:01:43.931Z","display_at":"2026-03-31T15:01:43.931Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"n2s72jfgjhz7","status":"investigating","body":"We have applied mitigations to a data store related to billing reports, and are seeing partial recovery to billing report generation. We continue to monitor for full recovery.","incident_id":"2yb2tvwny50l","created_at":"2026-03-31T14:59:06.066Z","updated_at":"2026-03-31T14:59:06.066Z","display_at":"2026-03-31T14:59:06.066Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"6k1qx136hdrd","status":"investigating","body":"We are seeing a high number of 500s due to timeouts across GitHub services. We are redeploying some of our core services and we expect that this allow us to recover.","incident_id":"2yb2tvwny50l","created_at":"2026-03-31T14:56:22.695Z","updated_at":"2026-03-31T14:56:22.695Z","display_at":"2026-03-31T14:56:22.695Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"z8938n1xwg41","status":"investigating","body":"We're continuing to see high failure rates on billing report generation, and are working on mitigations for a data store related to billing reports.","incident_id":"2yb2tvwny50l","created_at":"2026-03-31T14:39:15.197Z","updated_at":"2026-03-31T14:39:15.197Z","display_at":"2026-03-31T14:39:15.197Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"d46hydhg38qc","status":"investigating","body":"We're seeing issues related to metered billing reports, intermittently affecting metered usage graphs and reports on the billing page. We have identified an issue with a data store, and are working on mitigations.","incident_id":"2yb2tvwny50l","created_at":"2026-03-31T13:56:41.715Z","updated_at":"2026-03-31T13:56:41.715Z","display_at":"2026-03-31T13:56:41.715Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"5cywdg5ng2fs","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"2yb2tvwny50l","created_at":"2026-03-31T13:47:46.135Z","updated_at":"2026-03-31T13:47:46.135Z","display_at":"2026-03-31T13:47:46.135Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"c3ctbhdcvcc8","name":"Elevated delays in Actions workflow runs and Pull Request status updates","status":"resolved","created_at":"2026-03-30T13:02:28.248Z","updated_at":"2026-04-02T21:56:56.338Z","monitoring_at":"2026-03-30T13:20:00.602Z","resolved_at":"2026-03-30T13:25:38.683Z","impact":"minor","shortlink":"https://stspg.io/q4lmbp37zyrn","started_at":"2026-03-30T13:02:28.238Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"x0s8l499jf98","status":"resolved","body":"On March 30, 2026, between 10:11 UTC and 13:25 UTC, GitHub Actions experienced degraded performance. During this time, approximately 2.65% of workflow jobs triggered by pull request events experienced start delays exceeding 5 minutes. The issue was caused by replication lag on an internal database cluster used by Actions, which triggered write throttling in our database protection layer and slowed job queue processing. \u003cbr /\u003e\u003cbr /\u003eThe replication lag originated from planned maintenance to scale the internal database. Newly added database hosts triggered guardrails in the throttling layer, restricting write throughput. The incident was mitigated by excluding the new hosts from replication delay calculations.  \u003cbr /\u003e\u003cbr /\u003eTo prevent recurrence, we have updated our maintenance procedures to ensure new hosts are excluded from throttling assessments during scaling operations. Additionally, we are investing in automation to streamline this type of maintenance activity.","incident_id":"c3ctbhdcvcc8","created_at":"2026-03-30T13:25:38.683Z","updated_at":"2026-04-02T21:56:56.331Z","display_at":"2026-03-30T13:25:38.683Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"fw6415f78th3","status":"monitoring","body":"The degradation has been mitigated. We are monitoring to ensure stability.","incident_id":"c3ctbhdcvcc8","created_at":"2026-03-30T13:25:02.746Z","updated_at":"2026-03-30T13:25:02.746Z","display_at":"2026-03-30T13:25:02.746Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"3303ny844zp3","status":"monitoring","body":"The degradation affecting Actions and Pull Requests has been mitigated. We are monitoring to ensure stability.","incident_id":"c3ctbhdcvcc8","created_at":"2026-03-30T13:20:00.602Z","updated_at":"2026-03-30T13:20:00.602Z","display_at":"2026-03-30T13:20:00.602Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"q2f4rf9ln2zg","status":"investigating","body":"We are investigating reports of degraded performance for Actions and Pull Requests","incident_id":"c3ctbhdcvcc8","created_at":"2026-03-30T13:02:28.357Z","updated_at":"2026-03-30T13:02:28.357Z","display_at":"2026-03-30T13:02:28.357Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"hhtssxt0f5v2","name":"Pull Requests","status":"degraded_performance","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2026-04-28T15:58:26.154Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"br0l2tvcx85d","name":"Actions","status":"operational","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2026-04-28T17:08:59.360Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"9vyj2jwsk8by","name":"Incident with Copilot","status":"resolved","created_at":"2026-03-27T05:00:00.000Z","updated_at":"2026-03-27T18:42:58.384Z","monitoring_at":null,"resolved_at":"2026-03-27T05:00:00.000Z","impact":"none","shortlink":"https://stspg.io/41dqjldm4dwt","started_at":"2026-03-27T18:42:58.329Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"l2s81zpzc7yc","status":"resolved","body":"On March 27, 2026, from 02:30 to 04:56 UTC, a misconfiguration in our rate limiting system caused users on Copilot Free, Student, Pro, and Pro+ plans to experience unexpected rate limit errors. The configuration that was incorrectly applied was intended solely for internal staff testing of rate-limiting experiences. Copilot Business and Copilot Enterprise accounts were not affected.\n\nDuring this period, affected users received error messages instructing them to retry after a certain time. Approximately 32% of active Free users, 35% of active Student users, 46% of active Pro users, and 66% of active Pro+ users were affected.\n\nAfter identifying the root cause, we reverted the change and restored the expected rate limits. We are reviewing our deployment and validation processes to help ensure configurations used for internal testing cannot be inadvertently applied to production environments.","incident_id":"9vyj2jwsk8by","created_at":"2026-03-27T18:42:58.376Z","updated_at":"2026-03-27T18:42:58.376Z","display_at":"2026-03-27T05:00:00.000Z","affected_components":null,"deliver_notifications":false,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"kp06czybl7dw","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-03-24T20:18:06.873Z","updated_at":"2026-03-24T20:56:05.295Z","monitoring_at":null,"resolved_at":"2026-03-24T20:56:05.273Z","impact":"minor","shortlink":"https://stspg.io/dl16chvwsj7l","started_at":"2026-03-24T20:18:06.860Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"ll8pf12p3hjj","status":"resolved","body":"This incident has been resolved. Thank you for your patience and understanding as we addressed this issue. A detailed root cause analysis will be shared as soon as it is available.","incident_id":"kp06czybl7dw","created_at":"2026-03-24T20:56:05.273Z","updated_at":"2026-03-24T20:56:05.273Z","display_at":"2026-03-24T20:56:05.273Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"rhv8hn57fcsx","status":"investigating","body":"We are investigating elevated error rates affecting multiple GitHub services including Actions, Issues, Pull Requests, Webhooks, Codespaces, and login functionality. Some users may have experienced errors when accessing these features. Most services are now showing signs of recovery. We'll post another update by 21:00 UTC.","incident_id":"kp06czybl7dw","created_at":"2026-03-24T20:38:04.615Z","updated_at":"2026-03-24T20:38:04.615Z","display_at":"2026-03-24T20:38:04.615Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"4hmqz9cmjv17","status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","incident_id":"kp06czybl7dw","created_at":"2026-03-24T20:23:45.726Z","updated_at":"2026-03-24T20:23:45.726Z","display_at":"2026-03-24T20:23:45.726Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"59wmw9d1q5q9","status":"investigating","body":"Pull Requests is experiencing degraded performance. We are continuing to investigate.","incident_id":"kp06czybl7dw","created_at":"2026-03-24T20:23:44.534Z","updated_at":"2026-03-24T20:23:44.534Z","display_at":"2026-03-24T20:23:44.534Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vnqdlcc3ch0p","status":"investigating","body":"Webhooks is experiencing degraded performance. We are continuing to investigate.","incident_id":"kp06czybl7dw","created_at":"2026-03-24T20:20:11.069Z","updated_at":"2026-03-24T20:20:11.069Z","display_at":"2026-03-24T20:20:11.069Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"lfzmcnqf0fs7","status":"investigating","body":"We are investigating reports of degraded performance for Actions","incident_id":"kp06czybl7dw","created_at":"2026-03-24T20:18:06.963Z","updated_at":"2026-03-24T20:18:06.963Z","display_at":"2026-03-24T20:18:06.963Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"4230lsnqdsld","name":"Webhooks","status":"operational","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2026-04-23T17:10:29.626Z","position":2,"description":"Real time HTTP callbacks of user-generated and system events","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"kr09ddfgbfsf","name":"Issues","status":"operational","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2026-04-27T21:33:20.375Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"hhtssxt0f5v2","name":"Pull Requests","status":"degraded_performance","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2026-04-28T15:58:26.154Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"br0l2tvcx85d","name":"Actions","status":"operational","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2026-04-28T17:08:59.360Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"z7gsp4wd05c5","name":"Teams Github Notifications App is down","status":"resolved","created_at":"2026-03-24T16:59:27.656Z","updated_at":"2026-03-31T18:23:31.970Z","monitoring_at":null,"resolved_at":"2026-03-24T19:51:16.124Z","impact":"major","shortlink":"https://stspg.io/g3zv7t9bmccs","started_at":"2026-03-24T16:59:27.646Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"dnzzvz2pcbn8","status":"resolved","body":"On March 24, 2026, between 15:57 UTC and 19:51 UTC, the Microsoft Teams Integration and Teams Copilot Integration services were degraded and unable to deliver GitHub event notifications to Microsoft Teams. On average, the error rate was 37.4% and peaked at 90.1% of requests to the service -- approximately 19% of all integration installs failed to receive GitHub-to-Teams notifications in this time period.\u003cbr /\u003e\u003cbr /\u003eThis was due to an outage at one of our upstream dependencies, which caused HTTP 500 errors and connection resets for our Teams integration.\u003cbr /\u003e\u003cbr /\u003eWe coordinated with the relevant service teams, and the issue was resolved at 19:51 UTC when the upstream incident was mitigated.\u003cbr /\u003e\u003cbr /\u003eWe are working to update observability and runbooks to reduce time to mitigation for issues like this in the future.","incident_id":"z7gsp4wd05c5","created_at":"2026-03-24T19:51:16.124Z","updated_at":"2026-03-31T18:23:31.964Z","display_at":"2026-03-24T19:51:16.124Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"j1xqfjh1s531","status":"investigating","body":"We are experiencing degraded availability from Azure Teams APIs, which is impacting notifications from GitHub to Microsoft Teams. We are awaiting resolution from Azure.","incident_id":"z7gsp4wd05c5","created_at":"2026-03-24T18:50:56.177Z","updated_at":"2026-03-24T18:50:56.177Z","display_at":"2026-03-24T18:50:56.177Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"xcr1ym5qnrjf","status":"investigating","body":"We are experiencing degraded availability from Azure APIs, which is impacting notifications from GitHub to Microsoft Teams. We are working with Azure to resolve the issue.","incident_id":"z7gsp4wd05c5","created_at":"2026-03-24T17:43:14.237Z","updated_at":"2026-03-24T17:43:14.237Z","display_at":"2026-03-24T17:43:14.237Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"9zktcsr73nq6","status":"investigating","body":"We found an issue impacting notifications from GitHub to Microsoft Teams. We are working on mitigation and will keep users updated on progress towards mitigation.","incident_id":"z7gsp4wd05c5","created_at":"2026-03-24T17:09:57.765Z","updated_at":"2026-03-24T17:09:57.765Z","display_at":"2026-03-24T17:09:57.765Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"76xwwx1909k8","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"z7gsp4wd05c5","created_at":"2026-03-24T16:59:27.723Z","updated_at":"2026-03-24T16:59:27.723Z","display_at":"2026-03-24T16:59:27.723Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"7vwbds3snh28","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-03-22T09:08:38.800Z","updated_at":"2026-03-26T19:59:44.497Z","monitoring_at":null,"resolved_at":"2026-03-22T10:02:05.683Z","impact":"minor","shortlink":"https://stspg.io/l939pt92g9sj","started_at":"2026-03-22T09:08:38.793Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"51qh3460xrj2","status":"resolved","body":"On March 22, 2026, between 09:05 UTC and 10:02 UTC, users may have experienced intermittent errors and increased latency when performing Git http read operations. On average, the error rate was 3.84% and peaked at 15.55% of requests to the service. The issue was caused by elevated latency in an internal authentication service within one of our regional clusters. We mitigated the issue by redirecting traffic away from the affected cluster at 09:39 UTC, after which error rates returned to normal. The incident was fully resolved at 10:02 UTC.  \u003cbr /\u003e\u003cbr /\u003eWe are working to scale the authentication service and reduce our time to detection and mitigation of issues like this one in the future.","incident_id":"7vwbds3snh28","created_at":"2026-03-22T10:02:05.683Z","updated_at":"2026-03-26T19:59:44.493Z","display_at":"2026-03-22T10:02:05.683Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"441n64vzwwr4","status":"investigating","body":"We are investigating intermittently high latency and errors from Git operations.","incident_id":"7vwbds3snh28","created_at":"2026-03-22T09:27:39.768Z","updated_at":"2026-03-22T09:27:39.768Z","display_at":"2026-03-22T09:27:39.768Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"xtm3grhwqnc2","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"7vwbds3snh28","created_at":"2026-03-22T09:08:38.840Z","updated_at":"2026-03-22T09:08:38.840Z","display_at":"2026-03-22T09:08:38.840Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"9jyjvxkz735j","name":"Disruption with Copilot Coding Agent Sessions","status":"resolved","created_at":"2026-03-20T00:58:34.226Z","updated_at":"2026-03-25T14:54:45.078Z","monitoring_at":null,"resolved_at":"2026-03-20T01:58:40.445Z","impact":"major","shortlink":"https://stspg.io/5vnmx2tddlg2","started_at":"2026-03-20T00:58:34.220Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"fmgjhy686cz4","status":"resolved","body":"On March 19, 2026, between 01:05 UTC and 02:52 UTC, and again on March 20, 2026, between 00:42 UTC and 01:58 UTC, the Copilot Coding Agent service was degraded and users were unable to start new Copilot Agent sessions or view existing ones. During the first incident, the average error rate was ~53% and\u003cbr /\u003epeaked at ~93% of requests to the service. During the second incident, the average error rate was ~99%% and peaked at ~100%% of requests with significant retry amplification. Both incidents were caused by the same underlying system authentication issue that prevented the service from connecting to its\u003cbr /\u003ebacking datastore.\u003cbr /\u003e\u003cbr /\u003eWe mitigated each incident by rotating the affected credentials, which restored connectivity and returned error rates to normal. The mitigation time was 01:24. The second occurrence was due to an incomplete remediation of the first.\u003cbr /\u003e\u003cbr /\u003eWe are implementing automated monitoring for credential lifecycle events and improving operational processes to reduce our time to detection and mitigation of issues like this one in the future.","incident_id":"9jyjvxkz735j","created_at":"2026-03-20T01:58:40.445Z","updated_at":"2026-03-25T14:54:45.073Z","display_at":"2026-03-20T01:58:40.445Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"qzp5pfbt3wc2","status":"investigating","body":"We are rolling out our mitigation and are seeing recovery.","incident_id":"9jyjvxkz735j","created_at":"2026-03-20T01:26:19.374Z","updated_at":"2026-03-20T01:26:19.374Z","display_at":"2026-03-20T01:26:19.374Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"83g41fs10dff","status":"investigating","body":"We are seeing widespread issues starting and viewing Copilot Agent sessions. We understand the cause and are working on remediation.","incident_id":"9jyjvxkz735j","created_at":"2026-03-20T01:00:53.170Z","updated_at":"2026-03-20T01:00:53.170Z","display_at":"2026-03-20T01:00:53.170Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"cwttgn1xqxs2","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"9jyjvxkz735j","created_at":"2026-03-20T00:58:34.261Z","updated_at":"2026-03-20T00:58:34.261Z","display_at":"2026-03-20T00:58:34.261Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"p08cncy05m4k","name":"Git operations for users in the west coast are experiencing an increase in latency","status":"resolved","created_at":"2026-03-19T16:25:04.692Z","updated_at":"2026-03-31T17:01:09.320Z","monitoring_at":null,"resolved_at":"2026-03-20T00:05:15.572Z","impact":"minor","shortlink":"https://stspg.io/skszmdcydgwg","started_at":"2026-03-19T16:25:04.684Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"y8bmghzdtbd3","status":"resolved","body":"On March 19, 2026 between 16:10 UTC and 00:05 UTC (March 20), Git operations (clone, fetch, push) from the US west coast experienced elevated latency and degraded throughput. Users reported clone speeds dropping from typical speeds to under 1 MiB/s in extreme cases. The root cause was network transport link saturation at our Seattle edge site, where a fiber cut affecting our backbone transport resulted in saturation and packet loss. We had a planned scale-up in progress for the site that was accelerated to resolve the backbone capacity pressure. We also brought online additional edge capacity in a cloud region and redirected some users there. Current scale with the upgraded network capacity is sufficient to prevent reoccurrence, as we upgraded from 800Gbps to 3.2Tbps total capacity on this path. We will continue to monitor network health and respond to any further issues.","incident_id":"p08cncy05m4k","created_at":"2026-03-20T00:05:15.572Z","updated_at":"2026-03-31T17:01:09.314Z","display_at":"2026-03-20T00:05:15.572Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"c2ppj7wqqx1s","status":"investigating","body":"We have reached stability with git operations through our changes deployed today.","incident_id":"p08cncy05m4k","created_at":"2026-03-20T00:05:11.502Z","updated_at":"2026-03-20T00:05:11.502Z","display_at":"2026-03-20T00:05:11.502Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"dgjzcb2c39xv","status":"investigating","body":"We are seeing early signs of improvement. We are working on one more small change to further improve traffic routing on the west coast.","incident_id":"p08cncy05m4k","created_at":"2026-03-19T23:52:53.501Z","updated_at":"2026-03-19T23:52:53.501Z","display_at":"2026-03-19T23:52:53.501Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"z2lkrhv9t7hk","status":"investigating","body":"We have completed the rollout of our new network path and are monitoring its impact.","incident_id":"p08cncy05m4k","created_at":"2026-03-19T22:57:16.030Z","updated_at":"2026-03-19T22:57:16.030Z","display_at":"2026-03-19T22:57:16.030Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"m1b7qcsfs1qv","status":"investigating","body":"We are beginning the rollout of our new network path. During this change, users will continue to see higher latency from the west coast. We will provide another update when the rollout is complete.","incident_id":"p08cncy05m4k","created_at":"2026-03-19T21:59:53.554Z","updated_at":"2026-03-19T21:59:53.554Z","display_at":"2026-03-19T21:59:53.554Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"v6cxftd8mw1n","status":"investigating","body":"We are working to enable a new network path in the west coast to reduce load and will monitor the impact on latency for Git Operations","incident_id":"p08cncy05m4k","created_at":"2026-03-19T18:27:25.350Z","updated_at":"2026-03-19T18:27:25.350Z","display_at":"2026-03-19T18:27:25.350Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"5zbsm768901f","status":"investigating","body":"We are still seeing elevated latency for Git operations in the west coast and are continuing to investigate","incident_id":"p08cncy05m4k","created_at":"2026-03-19T17:49:51.787Z","updated_at":"2026-03-19T17:49:51.787Z","display_at":"2026-03-19T17:49:51.787Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"06yh5mbt0h6x","status":"investigating","body":"We are redirecting traffic back to our Seattle region and customers should see a decrease in latency for Git operations","incident_id":"p08cncy05m4k","created_at":"2026-03-19T17:01:25.420Z","updated_at":"2026-03-19T17:01:25.420Z","display_at":"2026-03-19T17:01:25.420Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"54wd0tgt9d1j","status":"investigating","body":"We are investigating reports of degraded performance for Git Operations","incident_id":"p08cncy05m4k","created_at":"2026-03-19T16:25:04.771Z","updated_at":"2026-03-19T16:25:04.771Z","display_at":"2026-03-19T16:25:04.771Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"8l4ygp009s5s","name":"Git Operations","status":"operational","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2026-03-20T00:05:15.542Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"3gflh59mjhmf","name":"Issues with Copilot Coding Agent","status":"resolved","created_at":"2026-03-19T13:44:15.731Z","updated_at":"2026-03-25T15:39:32.601Z","monitoring_at":null,"resolved_at":"2026-03-19T14:32:55.748Z","impact":"major","shortlink":"https://stspg.io/2wv8jbdygp2k","started_at":"2026-03-19T13:44:15.723Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"hd3mn3mn8h41","status":"resolved","body":"On March 19, 2026, between 01:05 UTC and 02:52 UTC, and again on March 20, 2026, between 00:42 UTC and 01:58 UTC, the Copilot Coding Agent service was degraded and users were unable to start new Copilot Agent sessions or view existing ones. During the first incident, the average error rate was ~53% and\u003cbr /\u003e peaked at ~93% of requests to the service. During the second incident, the average error rate was ~99%% and peaked at ~100%% of requests with significant retry amplification. Both incidents were caused by the same underlying system authentication issue that prevented the service from connecting to its\u003cbr /\u003e backing datastore.\u003cbr /\u003e \u003cbr /\u003e We mitigated each incident by rotating the affected credentials, which restored connectivity and returned error rates to normal. The mitigation time was 01:24. The second occurrence was due to an incomplete remediation of the first.\u003cbr /\u003e \u003cbr /\u003e We are implementing automated monitoring for credential lifecycle events and improving operational processes to reduce our time to detection and mitigation of issues like this one in the future.","incident_id":"3gflh59mjhmf","created_at":"2026-03-19T14:32:55.748Z","updated_at":"2026-03-25T15:39:32.596Z","display_at":"2026-03-19T14:32:55.748Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"mj6190f2whbp","status":"investigating","body":"Copilot is operating normally.","incident_id":"3gflh59mjhmf","created_at":"2026-03-19T14:06:44.686Z","updated_at":"2026-03-19T14:06:44.686Z","display_at":"2026-03-19T14:06:44.686Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"hg816k0cyjkp","status":"investigating","body":"We are investigating reports that Copilot Coding Agent session logs are not available in the UI.","incident_id":"3gflh59mjhmf","created_at":"2026-03-19T14:02:55.216Z","updated_at":"2026-03-19T14:02:55.216Z","display_at":"2026-03-19T14:02:55.216Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"8xq2kq9jkppc","status":"investigating","body":"Copilot is experiencing degraded performance. We are continuing to investigate.","incident_id":"3gflh59mjhmf","created_at":"2026-03-19T13:45:13.646Z","updated_at":"2026-03-19T13:45:13.646Z","display_at":"2026-03-19T13:45:13.646Z","affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vsb9gpj7lxpb","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"3gflh59mjhmf","created_at":"2026-03-19T13:44:15.770Z","updated_at":"2026-03-19T13:44:15.770Z","display_at":"2026-03-19T13:44:15.770Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"pjmpxvq2cmr2","name":"Copilot","status":"operational","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2026-04-23T17:03:00.903Z","position":10,"description":null,"showcase":true,"start_date":"2022-06-21","group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"djmkyscrj9jh","name":"Disruption with Copilot Coding Agent sessions","status":"resolved","created_at":"2026-03-19T02:05:59.173Z","updated_at":"2026-03-25T15:40:07.590Z","monitoring_at":null,"resolved_at":"2026-03-19T02:52:44.246Z","impact":"minor","shortlink":"https://stspg.io/qkddxwg74ymy","started_at":"2026-03-19T02:05:59.166Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"rbt1yw7f45gt","status":"resolved","body":"On March 19, 2026, between 01:05 UTC and 02:52 UTC, and again on March 20, 2026, between 00:42 UTC and 01:58 UTC, the Copilot Coding Agent service was degraded and users were unable to start new Copilot Agent sessions or view existing ones. During the first incident, the average error rate was ~53% and\u003cbr /\u003e peaked at ~93% of requests to the service. During the second incident, the average error rate was ~99%% and peaked at ~100%% of requests with significant retry amplification. Both incidents were caused by the same underlying system authentication issue that prevented the service from connecting to its\u003cbr /\u003e backing datastore.\u003cbr /\u003e \u003cbr /\u003e We mitigated each incident by rotating the affected credentials, which restored connectivity and returned error rates to normal. The mitigation time was 01:24. The second occurrence was due to an incomplete remediation of the first.\u003cbr /\u003e \u003cbr /\u003e We are implementing automated monitoring for credential lifecycle events and improving operational processes to reduce our time to detection and mitigation of issues like this one in the future.","incident_id":"djmkyscrj9jh","created_at":"2026-03-19T02:52:44.246Z","updated_at":"2026-03-25T15:40:07.582Z","display_at":"2026-03-19T02:52:44.246Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vqxpvmtf4qb1","status":"investigating","body":"We have rolled out our mitigation and are seeing recovery for Copilot Coding Agent sessions","incident_id":"djmkyscrj9jh","created_at":"2026-03-19T02:46:04.364Z","updated_at":"2026-03-19T02:46:04.364Z","display_at":"2026-03-19T02:46:04.364Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"bg6pxzz4vgts","status":"investigating","body":"We are seeing widespread issues starting and viewing Copilot Agent sessions. We have a hypothesis for the cause and are working on remediation.","incident_id":"djmkyscrj9jh","created_at":"2026-03-19T02:25:17.967Z","updated_at":"2026-03-19T02:25:17.967Z","display_at":"2026-03-19T02:25:17.967Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"6tvp8lgp3mft","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"djmkyscrj9jh","created_at":"2026-03-19T02:05:59.217Z","updated_at":"2026-03-19T02:05:59.217Z","display_at":"2026-03-19T02:05:59.217Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"49xnkj77r7vl","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-03-18T22:36:15.348Z","updated_at":"2026-04-07T19:22:44.273Z","monitoring_at":null,"resolved_at":"2026-03-19T01:44:01.056Z","impact":"minor","shortlink":"https://stspg.io/g773dw43b88l","started_at":"2026-03-18T22:36:15.341Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"c3rqpqwlp88w","status":"resolved","body":"On March 19, 2026 between 16:10 UTC and 00:05 UTC (March 20), Git operations (clone, fetch, push) from the US west coast experienced elevated latency and degraded throughput. Users reported clone speeds dropping from typical speeds to under 1 MiB/s in extreme cases. The root cause was network transport link saturation at our Seattle edge site, where a fiber cut affecting our backbone transport resulted in saturation and packet loss. We had a planned scale-up in progress for the site that was accelerated to resolve the backbone capacity pressure. We also brought online additional edge capacity in a cloud region and redirected some users there. Current scale with the upgraded network capacity is sufficient to prevent reoccurrence, as we upgraded from 800Gbps to 3.2Tbps total capacity on this path. We will continue to monitor network health and respond to any further issues.\u003cbr /\u003e\u003cbr /\u003eThis was the same incident declared in https://www.githubstatus.com/incidents/xs6xtcv196g7","incident_id":"49xnkj77r7vl","created_at":"2026-03-19T01:44:01.056Z","updated_at":"2026-04-07T19:22:44.266Z","display_at":"2026-03-19T01:44:01.056Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vdm6hg5q8bkf","status":"investigating","body":"We are seeing recovery in git operations for customers on the West Coast of the US.","incident_id":"49xnkj77r7vl","created_at":"2026-03-19T01:43:49.257Z","updated_at":"2026-03-19T01:43:49.257Z","display_at":"2026-03-19T01:43:49.257Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"6cnbnvz8spxf","status":"investigating","body":"We continue to investigate the slow performance of Git Operations affecting the US West Coast.","incident_id":"49xnkj77r7vl","created_at":"2026-03-19T00:56:37.483Z","updated_at":"2026-03-19T00:56:37.483Z","display_at":"2026-03-19T00:56:37.483Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"p58ng7svg4gp","status":"investigating","body":"We continue to investigate degraded performance for git operations from the US West Coast.","incident_id":"49xnkj77r7vl","created_at":"2026-03-19T00:10:19.498Z","updated_at":"2026-03-19T00:10:19.498Z","display_at":"2026-03-19T00:10:19.498Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"fcl4pq2grpbz","status":"investigating","body":"We are continuing to investigate degraded performance for git operations from the US West Coast.","incident_id":"49xnkj77r7vl","created_at":"2026-03-18T23:33:53.323Z","updated_at":"2026-03-18T23:33:53.323Z","display_at":"2026-03-18T23:33:53.323Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"zgtgsq2xj9zm","status":"investigating","body":"We are experiencing increased latency when performing git operations, especially large pushes and pulls from customers on the west coast of the US. We are not seeing an increase in failures. We are continuing to investigate.","incident_id":"49xnkj77r7vl","created_at":"2026-03-18T22:48:03.933Z","updated_at":"2026-03-18T22:48:03.933Z","display_at":"2026-03-18T22:48:03.933Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"kdrxmvpp6zc7","status":"investigating","body":"Git Operations is experiencing degraded performance. We are continuing to investigate.","incident_id":"49xnkj77r7vl","created_at":"2026-03-18T22:36:34.116Z","updated_at":"2026-03-18T22:36:34.116Z","display_at":"2026-03-18T22:36:34.116Z","affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"lccyj89b82g1","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"49xnkj77r7vl","created_at":"2026-03-18T22:36:15.387Z","updated_at":"2026-03-18T22:36:15.387Z","display_at":"2026-03-18T22:36:15.387Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"8l4ygp009s5s","name":"Git Operations","status":"operational","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2026-03-20T00:05:15.542Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"x1g78jx4sgfk","name":"Webhook delivery is delayed","status":"resolved","created_at":"2026-03-18T18:51:00.633Z","updated_at":"2026-03-19T21:15:58.025Z","monitoring_at":null,"resolved_at":"2026-03-18T19:46:38.809Z","impact":"minor","shortlink":"https://stspg.io/nq4nvbzkzfw3","started_at":"2026-03-18T18:51:00.625Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"z144v6ydmsc4","status":"resolved","body":"On March 18, 2026, between 18:18 UTC and 19:46 UTC all webhook deliveries experienced elevated latency. During this time, average delivery latency increased from a baseline of approximately 5 seconds to a peak of approximately 160 seconds. This was due to resource constraints in the webhook delivery pipeline, which caused queue backlog growth and increased delivery latency. We mitigated the incident by shifting traffic and adding capacity, after which webhook delivery latency returned to normal. We are working to improve capacity management and detection in the webhook delivery pipeline to help prevent similar issues in the future.","incident_id":"x1g78jx4sgfk","created_at":"2026-03-18T19:46:38.809Z","updated_at":"2026-03-19T21:15:58.021Z","display_at":"2026-03-18T19:46:38.809Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"cgt9n04ncgl4","status":"investigating","body":"We are seeing recovery and are continuing to monitor the latency for webhook deliveries","incident_id":"x1g78jx4sgfk","created_at":"2026-03-18T19:25:06.207Z","updated_at":"2026-03-18T19:25:06.207Z","display_at":"2026-03-18T19:25:06.207Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"1djbxhmzqtk8","status":"investigating","body":"We are investigating reports of degraded performance for Webhooks","incident_id":"x1g78jx4sgfk","created_at":"2026-03-18T18:51:00.710Z","updated_at":"2026-03-18T18:51:00.710Z","display_at":"2026-03-18T18:51:00.710Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"4230lsnqdsld","name":"Webhooks","status":"operational","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2026-04-23T17:10:29.626Z","position":2,"description":"Real time HTTP callbacks of user-generated and system events","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"2lh36qcrd7l6","name":"Errors starting and connecting to Codespaces","status":"resolved","created_at":"2026-03-16T15:01:06.899Z","updated_at":"2026-03-17T22:49:31.845Z","monitoring_at":null,"resolved_at":"2026-03-16T15:28:22.484Z","impact":"minor","shortlink":"https://stspg.io/5vpcvvyp7jdt","started_at":"2026-03-16T15:01:06.892Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"q23k7lffhq1k","status":"resolved","body":"On 16 March 2026, between 14:16 UTC and 15:18 UTC, Codespaces users encountered a download failure error message when starting newly created or resumed codespaces. At peak, 96% of the created or resumed codespaces were impacted. Active codespaces with a running VSCode environment were not affected.  \u003cbr /\u003e\u003cbr /\u003eThe error was a result of an API deployment issue with our VS Code remote experience dependency and was resolved by rolling back that deployment. We are working with our partners to reduce our incident engagement time, improve early detection before they impact our customers, and ensure safe rollout of similar changes in the future.","incident_id":"2lh36qcrd7l6","created_at":"2026-03-16T15:28:22.484Z","updated_at":"2026-03-17T22:49:31.840Z","display_at":"2026-03-16T15:28:22.484Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"2rcw767d9r78","status":"investigating","body":"Errors starting or resuming Codespaces have resolved.","incident_id":"2lh36qcrd7l6","created_at":"2026-03-16T15:27:21.684Z","updated_at":"2026-03-16T15:27:21.684Z","display_at":"2026-03-16T15:27:21.684Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vcy8tjr1zl93","status":"investigating","body":"We are investigating reports of users experiencing errors when starting or connecting to Codespaces. Some users may be unable to access their development environments during this time. We are working to identify the root cause and will implement a fix as soon as possible.","incident_id":"2lh36qcrd7l6","created_at":"2026-03-16T15:06:35.960Z","updated_at":"2026-03-16T15:06:35.960Z","display_at":"2026-03-16T15:06:35.960Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"jtx5ftzdkzdv","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"2lh36qcrd7l6","created_at":"2026-03-16T15:01:06.937Z","updated_at":"2026-03-16T15:01:06.937Z","display_at":"2026-03-16T15:01:06.937Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"xsxcyn25nfmq","name":"Degraded performance for various services","status":"resolved","created_at":"2026-03-13T15:12:48.287Z","updated_at":"2026-03-17T13:27:46.454Z","monitoring_at":null,"resolved_at":"2026-03-13T16:15:33.784Z","impact":"minor","shortlink":"https://stspg.io/xrr4ln3tbt88","started_at":"2026-03-13T15:12:48.279Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"7vxwfzqwqnml","status":"resolved","body":"On March 13, 2026, between 13:35 UTC and 16:02 UTC, a configuration change to an internal authorization service reduced its processing capacity below what was needed during peak traffic. This caused intermittent timeouts when other GitHub services checked user permissions, resulting in four to five waves of errors over roughly two hours and forty minutes. In total, 0.4% of users were denied access to actions they were authorized to perform. \u003cbr /\u003e\u003cbr /\u003eThe root cause was a resource right-sizing change deployed to the authorization service the previous day. It reduced CPU allocation below what was required at peak, causing the service's network gateway to throttle under load. Because the change was deployed after peak traffic on March 12, the reduced capacity wasn't surfaced until the next day's peak. \u003cbr /\u003e\u003cbr /\u003eThe incident was mitigated by manually scaling up the authorization service and reverting the configuration change. \u003cbr /\u003e\u003cbr /\u003e  \u003cbr /\u003eTo prevent recurrence, we are adding further resource utilization monitors across our entire stack to detect throttling and improving error handling so transient infrastructure timeouts are distinguished from authorization failures, enabling quicker detection of the root issue.","incident_id":"xsxcyn25nfmq","created_at":"2026-03-13T16:15:33.784Z","updated_at":"2026-03-17T13:27:46.450Z","display_at":"2026-03-13T16:15:33.784Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vrtlrls30m99","status":"investigating","body":"We have deployed mitigations and are actively monitoring for recovery. We'll post another update by 17:00 UTC.","incident_id":"xsxcyn25nfmq","created_at":"2026-03-13T16:02:54.830Z","updated_at":"2026-03-13T16:02:54.830Z","display_at":"2026-03-13T16:02:54.830Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"kybtxc51vmwn","status":"investigating","body":"We are investigating intermittent performance degradation affecting Actions, Feeds, Issues, Package Registry, Profiles, Registry Metadata, Star, and User Dashboard. Users may experience elevated error rates and slower response times when accessing these services. We have identified a potential cause and are implementing mitigations to restore normal service. We'll post another update by 16:15 UTC.","incident_id":"xsxcyn25nfmq","created_at":"2026-03-13T15:47:51.092Z","updated_at":"2026-03-13T15:47:51.092Z","display_at":"2026-03-13T15:47:51.092Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"qynj87ll891t","status":"investigating","body":"Packages is experiencing degraded performance. We are continuing to investigate.","incident_id":"xsxcyn25nfmq","created_at":"2026-03-13T15:20:01.375Z","updated_at":"2026-03-13T15:20:01.375Z","display_at":"2026-03-13T15:20:01.375Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"st3j38cctv9l","name":"Packages","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"n38f5j7c00bg","status":"investigating","body":"We are investigating reports of issues with service(s): Actions, Feeds, Issues, Profiles, Registry Metadata, Star, User Dashboard. We will continue to keep users updated on progress towards mitigation.","incident_id":"xsxcyn25nfmq","created_at":"2026-03-13T15:14:03.343Z","updated_at":"2026-03-13T15:14:03.343Z","display_at":"2026-03-13T15:14:03.343Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"7b1wyy9k3bg6","status":"investigating","body":"We are investigating reports of degraded performance for Actions and Issues","incident_id":"xsxcyn25nfmq","created_at":"2026-03-13T15:12:48.376Z","updated_at":"2026-03-13T15:12:48.376Z","display_at":"2026-03-13T15:12:48.376Z","affected_components":[{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"kr09ddfgbfsf","name":"Issues","status":"operational","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2026-04-27T21:33:20.375Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"br0l2tvcx85d","name":"Actions","status":"operational","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2026-04-28T17:08:59.360Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false},{"id":"st3j38cctv9l","name":"Packages","status":"operational","created_at":"2019-11-13T18:02:40.064Z","updated_at":"2026-04-27T21:33:20.412Z","position":8,"description":"API requests and webhook delivery for GitHub Packages","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"0s7ppykbyvqz","name":"Degraded Codespaces experience","status":"resolved","created_at":"2026-03-12T13:06:54.989Z","updated_at":"2026-03-16T20:39:15.121Z","monitoring_at":null,"resolved_at":"2026-03-12T18:53:33.481Z","impact":"minor","shortlink":"https://stspg.io/lq4q8bqfnkqn","started_at":"2026-03-12T13:06:54.982Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"g97yzfttd4y2","status":"resolved","body":"On March 12, 2026, between 01:00 UTC and 18:53 UTC, users saw failures downloading extensions within created or resumed codespaces. Users would see an error when attempting to use an extension within VS Code. Active codespaces with extensions already downloaded were not impacted.\u003cbr /\u003e\u003cbr /\u003eThe extensions download failures were the result of a change introduced in our extension dependency and was resolved by updating the configuration of how those changes affect requests from Codespaces. We are enhancing observability and alerting of critical issues within regular codespace operations to better detect and mitigate similar issues in the future.","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T18:53:33.481Z","updated_at":"2026-03-16T20:39:15.116Z","display_at":"2026-03-12T18:53:33.481Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"h1qjzqqry72q","status":"investigating","body":"Codespaces IPs are no longer being blocked from Visual Studio Marketplace operations and we are monitoring for full recovery","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T17:59:56.435Z","updated_at":"2026-03-12T17:59:56.435Z","display_at":"2026-03-12T17:59:56.435Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"snw63qwmddmg","status":"investigating","body":"We're seeing intermittent failures downloading from the extension marketplace from codespaces, caused by IP blocks for some codespaces. We're working to remove those blocks.","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T17:20:20.620Z","updated_at":"2026-03-12T17:20:20.620Z","display_at":"2026-03-12T17:20:20.620Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"kv97c68ydxj3","status":"investigating","body":"We're seeing intermittent failures downloading from the extension marketplace from codespaces and are investigating.","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T16:09:52.747Z","updated_at":"2026-03-12T16:09:52.747Z","display_at":"2026-03-12T16:09:52.747Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"yvgt8l0ddkvw","status":"investigating","body":"We're seeing partial recovery for the issue affecting extension installation in newly created Codespaces. Some users may still experience degraded functionality where extensions hit errors. The team continues to investigate the root cause while monitoring the recovery.","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T15:08:15.929Z","updated_at":"2026-03-12T15:08:15.929Z","display_at":"2026-03-12T15:08:15.929Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"lkp4v5t5zdzr","status":"investigating","body":"We have deployed a fix for the issue affecting extension installation in newly created Codespaces. New Codespaces are now being created with working extensions. We'll post another update by 15:30 UTC.","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T14:29:31.025Z","updated_at":"2026-03-12T14:29:31.025Z","display_at":"2026-03-12T14:29:31.025Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"cwqndh33r1th","status":"investigating","body":"We are continuing to investigate an issue where extensions fail to install in newly created Codespaces. Users can create and access Codespaces, but extensions will not be operational, resulting in a degraded experience. The team is working on a fix. All newly created Codespaces are affected. We'll post another update by 15:00 UTC.","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T13:50:06.507Z","updated_at":"2026-03-12T13:50:06.507Z","display_at":"2026-03-12T13:50:06.507Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"r4fv48sgn4pt","status":"investigating","body":"We're investigating an issue where extensions fail to install in newly created Codespaces. Users can still create and access Codespaces, but extensions will not be operational, resulting in a degraded development experience. Our team is actively working to identify and resolve the root cause. We'll post another update by 14:00 UTC.","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T13:07:45.295Z","updated_at":"2026-03-12T13:07:45.295Z","display_at":"2026-03-12T13:07:45.295Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"0d3lx2g6fy6s","status":"investigating","body":"We are investigating reports of degraded performance for Codespaces","incident_id":"0s7ppykbyvqz","created_at":"2026-03-12T13:06:55.060Z","updated_at":"2026-03-12T13:06:55.060Z","display_at":"2026-03-12T13:06:55.060Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"h2ftsgbw7kmk","name":"Codespaces","status":"operational","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2026-04-23T15:02:29.440Z","position":11,"description":"Orchestration and Compute for GitHub Codespaces","showcase":true,"start_date":"2021-08-11","group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"02z04m335tvv","name":"Actions failures to download (401 Unauthorized)","status":"resolved","created_at":"2026-03-12T04:46:20.069Z","updated_at":"2026-03-16T17:22:02.289Z","monitoring_at":"2026-03-12T06:02:05.184Z","resolved_at":"2026-03-12T06:02:07.228Z","impact":"minor","shortlink":"https://stspg.io/8t1nfjvmmtqj","started_at":"2026-03-12T04:46:20.061Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"d4h67lxcbbht","status":"resolved","body":"On March 12, 2026 between 02:30 and 06:02 UTC some GitHub Apps were unable to mint server to server tokens, resulting in 401 Unauthorized errors. During the outage window, ~1.3% of requests resulted in 401 errors incorrectly. This manifested in GitHub Actions jobs failing to download tarballs, as well as failing to mint fine-grained tokens. During this period, approximately 5% of Actions jobs were impacted \u003cbr /\u003e\u003cbr /\u003eThe root cause was a failure with the authentication service’s token cache layer, a newly created secondary cache layer backed by Redis – caused by Kubernetes control plane instability, leading to an inability to read certain tokens which resulted in 401 errors. The mitigation was to fallback reads to the primary cache layer backed by mysql. As permanent mitigations, we have made changes to how we deploy redis to not rely on the Kubernetes control plane and maintain service availability during similar failure modes.  We also improved alerting to reduce overall impact time from similar failures.  \u003cbr /\u003e","incident_id":"02z04m335tvv","created_at":"2026-03-12T06:02:07.228Z","updated_at":"2026-03-16T17:22:02.284Z","display_at":"2026-03-12T06:02:07.228Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"7vr99lqmfbf4","status":"monitoring","body":"Actions is operating normally.","incident_id":"02z04m335tvv","created_at":"2026-03-12T06:02:05.184Z","updated_at":"2026-03-12T06:02:05.184Z","display_at":"2026-03-12T06:02:05.184Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"71jdvdmyfj7m","status":"investigating","body":"We are continuing investigation of reports of degraded performance for Actions and GitHub Apps","incident_id":"02z04m335tvv","created_at":"2026-03-12T05:40:13.649Z","updated_at":"2026-03-12T05:40:13.649Z","display_at":"2026-03-12T05:40:13.649Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"z2kysldtv1mp","status":"investigating","body":"We are investigating reports of degraded performance for Actions","incident_id":"02z04m335tvv","created_at":"2026-03-12T04:46:20.134Z","updated_at":"2026-03-12T04:46:20.134Z","display_at":"2026-03-12T04:46:20.134Z","affected_components":[{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"br0l2tvcx85d","name":"Actions","status":"operational","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2026-04-28T17:08:59.360Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"lw6j95nyw3py","name":"Disruption with some GitHub services","status":"resolved","created_at":"2026-03-12T01:54:18.324Z","updated_at":"2026-04-08T17:43:07.790Z","monitoring_at":null,"resolved_at":"2026-03-12T02:45:55.801Z","impact":"minor","shortlink":"https://stspg.io/tg49qcr0n1lc","started_at":"2026-03-12T01:54:18.318Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"ptjsy6xwc6vq","status":"resolved","body":"Between 01:36 and 08:11 UTC on Thursday March 12, GitHub.com experienced elevated error rates across Git operations, web requests, and related services. During a planned infrastructure upgrade, a configuration issue caused newly provisioned Kubernetes nodes to run an incompatible version of etcd, which disrupted cluster consensus across several production clusters. This led to intermittent 5XX errors on git push, git clone, and page loads. Deployments were paused for the duration of the incident.\u003cbr /\u003e\u003cbr /\u003eOnce the incompatible nodes were identified, they were removed and cluster consensus was restored. A validation deploy confirmed all systems were healthy before normal operations resumed.\u003cbr /\u003e\u003cbr /\u003eTo prevent recurrence, we are adding programmatic enforcement of version compatibility during node replacements, implementing monitoring to detect split-brain conditions earlier, and updating our recovery tooling to reduce restoration time.","incident_id":"lw6j95nyw3py","created_at":"2026-03-12T02:45:55.801Z","updated_at":"2026-04-08T17:43:07.785Z","display_at":"2026-03-12T02:45:55.801Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"01xmcsz8ksdh","status":"investigating","body":"We've identified the root cause and are working on resolving the underlying issue. Some users may have encountered intermittent failures and errors. We're continuing to see reduced error rates.","incident_id":"lw6j95nyw3py","created_at":"2026-03-12T02:44:51.224Z","updated_at":"2026-03-12T02:44:51.224Z","display_at":"2026-03-12T02:44:51.224Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vl33t77ncytt","status":"investigating","body":"We are investigating elevated error rates. Error rates are now decreasing and we're continuing to monitor the situation.","incident_id":"lw6j95nyw3py","created_at":"2026-03-12T02:13:26.637Z","updated_at":"2026-03-12T02:13:26.637Z","display_at":"2026-03-12T02:13:26.637Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"g7351jh00lbv","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"lw6j95nyw3py","created_at":"2026-03-12T01:54:18.364Z","updated_at":"2026-03-12T01:54:18.364Z","display_at":"2026-03-12T01:54:18.364Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"78s4l7y4tn2v","name":"Degraded experience with Copilot Code Review","status":"resolved","created_at":"2026-03-11T14:25:17.532Z","updated_at":"2026-03-13T20:03:40.829Z","monitoring_at":"2026-03-11T14:28:04.402Z","resolved_at":"2026-03-11T15:53:15.796Z","impact":"minor","shortlink":"https://stspg.io/twfx8xg9nf33","started_at":"2026-03-11T14:25:17.526Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"0rljcgxhy7b2","status":"resolved","body":"On March 11, 2026, between 13:00 UTC and 15:23 UTC the Copilot Code Review service was degraded and experienced longer than average review times. On average, Copilot Code Review requests took 4 minutes and peaked at just under 8 minutes. This was due to hitting worker capacity limits and CPU throttling. We mitigated the incident by increasing partitions, and we are improving our resource monitoring to identify potential issues sooner.","incident_id":"78s4l7y4tn2v","created_at":"2026-03-11T15:53:15.796Z","updated_at":"2026-03-13T20:03:40.824Z","display_at":"2026-03-11T15:53:15.796Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"h72721zccrcs","status":"monitoring","body":"Copilot Code Review queue processing has returned to normal levels.","incident_id":"78s4l7y4tn2v","created_at":"2026-03-11T15:53:07.546Z","updated_at":"2026-03-11T15:53:07.546Z","display_at":"2026-03-11T15:53:07.546Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"nbbk9nmqk0pv","status":"monitoring","body":"We experienced degraded performance with Copilot Code Review starting at 14:01 UTC. Customers experienced extended review times and occasional failures. Some extended processing times may continue briefly. We are monitoring for full recovery. We'll post another update by 16:30 UTC.","incident_id":"78s4l7y4tn2v","created_at":"2026-03-11T15:31:21.866Z","updated_at":"2026-03-11T15:31:21.866Z","display_at":"2026-03-11T15:31:21.866Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"6qshznhqrzl0","status":"monitoring","body":"We are investigating degraded performance with Copilot Code Review. Customers may experience extended review times or occasional failures. We are seeing signs of improvement as our team works to restore normal service. We'll post another update by 15:30 UTC.","incident_id":"78s4l7y4tn2v","created_at":"2026-03-11T14:28:04.402Z","updated_at":"2026-03-11T14:28:04.402Z","display_at":"2026-03-11T14:28:04.402Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"z1jk8c7h4qqm","status":"investigating","body":"We are investigating reports of impacted performance for some GitHub services.","incident_id":"78s4l7y4tn2v","created_at":"2026-03-11T14:25:17.570Z","updated_at":"2026-03-11T14:25:17.570Z","display_at":"2026-03-11T14:25:17.570Z","affected_components":null,"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"zy1gjfc6x2z9","name":"Incident with API Requests","status":"resolved","created_at":"2026-03-11T14:37:31.222Z","updated_at":"2026-03-25T20:03:16.304Z","monitoring_at":null,"resolved_at":"2026-03-11T15:02:23.694Z","impact":"minor","shortlink":"https://stspg.io/vg3jh54ynrp5","started_at":"2026-03-11T14:37:31.214Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"jlbtfxqjcssd","status":"resolved","body":"On March 11, 2026, between 14:25 UTC and 14:34 UTC, the REST API platform was degraded, resulting in increased error rates and request timeouts. REST API 5xx error rates peaked at ~5% during the incident window with two distinct spikes: the first impacting REST services broadly, and the second driven by sustained timeouts on a subset of endpoints. \u003cbr /\u003e\u003cbr /\u003eThe incident was caused by a performance degradation in our data layer, which resulted in increased query latency across dependent services. Most services recovered quickly after the initial spike, but resource contention caused sustained 5xx errors due to how certain endpoints responded to the degraded state. \u003cbr /\u003e\u003cbr /\u003eA fix addressing the behavior that prolonged impact has already been shipped. We are continuing to work to resolve the primary contributing factor of the degradation and to implement safeguards against issues causing cascading impact in the future.","incident_id":"zy1gjfc6x2z9","created_at":"2026-03-11T15:02:23.694Z","updated_at":"2026-03-25T20:03:16.298Z","display_at":"2026-03-11T15:02:23.694Z","affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"g1ych5jmmmdt","status":"investigating","body":"We are investigating elevated timeouts that affected GitHub API requests. The incident began at 14:37 UTC. Some users experienced slower response times and request failures. System metrics have returned to normal levels, and we are now investigating the root cause to prevent recurrence.","incident_id":"zy1gjfc6x2z9","created_at":"2026-03-11T15:02:17.591Z","updated_at":"2026-03-11T15:02:17.591Z","display_at":"2026-03-11T15:02:17.591Z","affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"q2r05qcltkg6","status":"investigating","body":"We are investigating reports of degraded performance for API Requests","incident_id":"zy1gjfc6x2z9","created_at":"2026-03-11T14:37:31.315Z","updated_at":"2026-03-11T14:37:31.315Z","display_at":"2026-03-11T14:37:31.315Z","affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"brv1bkgrwx7q","name":"API Requests","status":"operational","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2026-03-11T15:02:23.656Z","position":4,"description":"Requests for GitHub APIs","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"8jx0t5c136x2","name":"Incident With Webhooks","status":"resolved","created_at":"2026-03-10T15:00:00.000Z","updated_at":"2026-03-18T16:06:25.575Z","monitoring_at":null,"resolved_at":"2026-03-10T23:00:00.000Z","impact":"none","shortlink":"https://stspg.io/9dy0pjrt1hwh","started_at":"2026-03-10T23:00:00.000Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"l44wgmpl0n3l","status":"resolved","body":"On March 10, 2026, between 23:00 UTC and 23:40 UTC, the Webhooks service was degraded and ~6% of users experienced intermittent errors when accessing webhook delivery history, retrying webhook deliveries, and listing webhooks via the UI and API. Approximately 0.37% of requests resulted in errors, while at peak 0.5% of requests resulted in errors.\n\nThis was due to unhealthy infrastructure. We mitigated the incident by redeploying affected services, after which service health returned to normal.\n\nWe are working to improve detection of unhealthy infrastructure and strengthen service safeguards to reduce time to detect and mitigate similar issues in the future.","incident_id":"8jx0t5c136x2","created_at":"2026-03-18T16:04:49.333Z","updated_at":"2026-03-18T16:06:25.503Z","display_at":"2026-03-10T23:00:00.000Z","affected_components":null,"deliver_notifications":false,"custom_tweet":null,"tweet_id":null}],"components":[],"reminder_intervals":null},{"id":"7k92rcz4f1zq","name":"Incident with Webhooks","status":"resolved","created_at":"2026-03-09T15:50:24.508Z","updated_at":"2026-03-17T21:50:37.970Z","monitoring_at":null,"resolved_at":"2026-03-09T17:03:40.901Z","impact":"minor","shortlink":"https://stspg.io/pqgzxr03wx63","started_at":"2026-03-09T15:50:24.501Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"ktgpqv2v3tx8","status":"resolved","body":"On March 9, 2026, between 15:03 and 20:52 UTC, the Webhooks API experienced was degraded, resulted in higher average latency on requests and in certain cases error responses. Approximately 0.6% of total requests exceeded the normal latency threshold of 3s, while 0.4% of requests resulted in 500 errors. At peak, 2.0% experienced latency greater than 3 seconds and 2.8% of requests returned 500 errors.\u003cbr /\u003e\u003cbr /\u003eThe issue was caused by a noisy actor that led to resource contention on the Webhooks API service. We mitigated the issue initially by increasing CPU resources for the Webhooks API service, and ultimately applied lower rate limiting thresholds to the noisy actor to prevent further impact to other users.\u003cbr /\u003e\u003cbr /\u003eWe are working to improve monitoring to more quickly ascertain noisy traffic and will continue to improve our rate-limiting mechanisms to help prevent similar issues in the future.","incident_id":"7k92rcz4f1zq","created_at":"2026-03-09T17:03:40.901Z","updated_at":"2026-03-17T21:50:37.966Z","display_at":"2026-03-09T17:03:40.901Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"vn4k0g85hgkd","status":"investigating","body":"Webhooks is operating normally.","incident_id":"7k92rcz4f1zq","created_at":"2026-03-09T17:03:39.350Z","updated_at":"2026-03-09T17:03:39.350Z","display_at":"2026-03-09T17:03:39.350Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"yh8t9vsggc6k","status":"investigating","body":"We are experiencing latency on the API and UI endpoints. We are working to resolve the issue.","incident_id":"7k92rcz4f1zq","created_at":"2026-03-09T15:56:01.322Z","updated_at":"2026-03-09T15:56:01.322Z","display_at":"2026-03-09T15:56:01.322Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"cz8hnfc2mfxb","status":"investigating","body":"We are investigating reports of degraded performance for Webhooks","incident_id":"7k92rcz4f1zq","created_at":"2026-03-09T15:50:24.582Z","updated_at":"2026-03-09T15:50:24.582Z","display_at":"2026-03-09T15:50:24.582Z","affected_components":[{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"4230lsnqdsld","name":"Webhooks","status":"operational","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2026-04-23T17:10:29.626Z","position":2,"description":"Real time HTTP callbacks of user-generated and system events","showcase":true,"start_date":null,"group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null},{"id":"tp8m3544w2g8","name":"Incident with Codespaces","status":"resolved","created_at":"2026-03-09T03:04:04.091Z","updated_at":"2026-03-10T18:32:23.101Z","monitoring_at":null,"resolved_at":"2026-03-09T03:51:42.260Z","impact":"minor","shortlink":"https://stspg.io/tvxhncnsvhlt","started_at":"2026-03-09T03:04:04.085Z","page_id":"kctbh9vrtdwd","incident_updates":[{"id":"504mlkrds8y9","status":"resolved","body":"On March 9, 2026, between 01:23 UTC and 03:25 UTC, users attempting to create or resume codespaces in the Australia East region experienced elevated failures, peaking at a 100% failure rate for this region. Codespaces in other regions were not affected.\u003cbr /\u003e\u003cbr /\u003eThe create and resume failures were caused by degraded network connectivity between our control plane services and the VMs hosting the codespaces. This was resolved by redirecting traffic to an alternate site within the region. While we are addressing the core network infrastructure issue, we have also improved our observability of components in this area to improve detection. This will also enable our existing automated failovers to cover this failure mode. These changes will prevent or significantly reduce the time any similar incident causes user impact.","incident_id":"tp8m3544w2g8","created_at":"2026-03-09T03:51:42.260Z","updated_at":"2026-03-10T18:32:23.097Z","display_at":"2026-03-09T03:51:42.260Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"operational"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"m6zl3mg2mmrm","status":"investigating","body":"This incident has been resolved. New Codespace creation requests are now completing successfully.","incident_id":"tp8m3544w2g8","created_at":"2026-03-09T03:51:32.960Z","updated_at":"2026-03-09T03:51:32.960Z","display_at":"2026-03-09T03:51:32.960Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"g16vt5cg13d3","status":"investigating","body":"We are seeing recovery, with the failure rate for new Codespace creation requests dropping from 5% to about 3%.","incident_id":"tp8m3544w2g8","created_at":"2026-03-09T03:32:35.715Z","updated_at":"2026-03-09T03:32:35.715Z","display_at":"2026-03-09T03:32:35.715Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"qhsmsvg7ssb4","status":"investigating","body":"We are seeing about 5% of new Codespace creation requests failing. We are investigating the root cause and identifying the impacted regions.","incident_id":"tp8m3544w2g8","created_at":"2026-03-09T03:04:16.325Z","updated_at":"2026-03-09T03:04:16.325Z","display_at":"2026-03-09T03:04:16.325Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null},{"id":"2l7f3zzmvprl","status":"investigating","body":"We are investigating reports of degraded performance for Codespaces","incident_id":"tp8m3544w2g8","created_at":"2026-03-09T03:04:04.148Z","updated_at":"2026-03-09T03:04:04.148Z","display_at":"2026-03-09T03:04:04.148Z","affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"partial_outage"}],"deliver_notifications":true,"custom_tweet":null,"tweet_id":null}],"components":[{"id":"h2ftsgbw7kmk","name":"Codespaces","status":"operational","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2026-04-23T15:02:29.440Z","position":11,"description":"Orchestration and Compute for GitHub Codespaces","showcase":true,"start_date":"2021-08-11","group_id":null,"page_id":"kctbh9vrtdwd","group":false,"only_show_if_degraded":false}],"reminder_intervals":null}]}