{
  "page": {
    "id": "kctbh9vrtdwd",
    "name": "GitHub",
    "url": "https://web.archive.org/web/20240502121617/https://www.githubstatus.com"
  },
  "status": {
    "indicator": "none",
    "description": "All Systems Operational"
  },
  "components": [{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2024-04-24T16:13:06.507Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2024-04-24T16:12:42.107Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2024-04-24T16:13:07.705Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Visit www.githubstatus.com for more information","created_at":"2018-12-05T19:39:40.838Z","updated_at":"2022-09-07T00:08:33.519Z","position":4,"description":null,"showcase":false,"start_date":null,"id":"0l2p9nhqnxpd","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2024-04-24T16:13:44.528Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pull Requests","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2024-04-24T18:20:14.311Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":false,"start_date":null,"id":"hhtssxt0f5v2","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2024-04-24T16:13:25.696Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Packages","created_at":"2019-11-13T18:02:40.064Z","updated_at":"2024-01-09T14:37:18.000Z","position":8,"description":"API requests and webhook delivery for GitHub Packages","showcase":false,"start_date":null,"id":"st3j38cctv9l","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pages","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2024-04-10T19:03:05.749Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":false,"start_date":null,"id":"vg70hn9s2tyj","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Codespaces","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2024-04-18T18:47:26.803Z","position":10,"description":"Orchestration and Compute for GitHub Codespaces","showcase":false,"start_date":"2021-08-11T00:00:00.000Z","id":"h2ftsgbw7kmk","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2024-04-17T00:48:52.072Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}],
  "incidents": [{"name":"We are investigating reports of degraded performance.","status":"resolved","created_at":"2024-04-26T15:10:10.698Z","updated_at":"2024-04-30T17:59:20.697Z","monitoring_at":null,"resolved_at":"2024-04-26T16:49:38.810Z","impact":"minor","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/zynx544zbs5y","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-26T15:10:10.692Z","reminder_intervals":null,"id":"84bxn520qg2m","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On April 26th, 2024, we received several internal reports of GitHub staff being unable to login using their Passkeys. Out of an abundance of caution, we updated our public status to yellow. Further investigation discovered that a background job intended to clean up stale authenticated device records was mistakenly removing associated passkeys.  As a result, some passkey registrations were incorrectly removed from some user accounts.\u003cbr /\u003eThe impact from the bug in the background job was limited to a small number of passkey users (\u003c0.5%). We have since resolved the bug in question and are working to notify all affected users.","created_at":"2024-04-26T16:49:38.810Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-30T17:59:20.694Z","display_at":"2024-04-26T16:49:38.810Z","deliver_notifications":true,"tweet_id":null,"id":"zr0p5p6drjh7","incident_id":"84bxn520qg2m","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"The issue appears to be limited in scope to a few internal users, without any reports of issues from outside GitHub. We are adding additional logging to our WebAuthn flow to detect this in the future. If you cannot use your mobile passkey to sign in, please contact support or reach out to us in https://github.com/orgs/community/discussions/67791","created_at":"2024-04-26T16:49:19.889Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-26T16:49:19.889Z","display_at":"2024-04-26T16:49:19.889Z","deliver_notifications":true,"tweet_id":null,"id":"9xb42z7l8szh","incident_id":"84bxn520qg2m","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Sign in to GitHub.com using a passkey from a mobile device is currently failing. Users may see an error message saying that passkey sign in failed, or may not see any passkeys available after signing in with their password.\u003cbr /\u003eThis issue impacts both GitHub.com on mobile devices as well as cross-device authentication where the phone's passkey is used to authenticate on a desktop browser.\u003cbr /\u003eTo workaround this issue, use your password and the 2FA method you setup prior to setting up your passkey, either TOTP or SMS.","created_at":"2024-04-26T15:10:41.929Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-26T15:10:41.929Z","display_at":"2024-04-26T15:10:41.929Z","deliver_notifications":true,"tweet_id":null,"id":"1pnhjs8sykw1","incident_id":"84bxn520qg2m","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2024-04-26T15:10:10.731Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-26T15:10:10.731Z","display_at":"2024-04-26T15:10:10.731Z","deliver_notifications":true,"tweet_id":null,"id":"w84lxxv97n9r","incident_id":"84bxn520qg2m","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"Incident with Pull Requests","status":"resolved","created_at":"2024-04-24T17:40:48.299Z","updated_at":"2024-04-30T13:05:34.603Z","monitoring_at":null,"resolved_at":"2024-04-24T18:20:14.342Z","impact":"minor","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/k6h8wphh03nq","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-24T17:40:48.294Z","reminder_intervals":null,"id":"lg4ndssz2n7l","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"This incident was caused by a mitigation for \u003ca href=\"https://www.githubstatus.com/incidents/7txcg0j03kkg\"\u003ethis incident\u003c/a\u003e. Please follow the link for the incident summary.","created_at":"2024-04-24T18:20:14.342Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-30T13:05:34.599Z","display_at":"2024-04-24T18:20:14.342Z","deliver_notifications":true,"tweet_id":null,"id":"bxzpwyc1r84p","incident_id":"lg4ndssz2n7l","custom_tweet":null,"affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"The previous mitigation has been rolled back and updates to the pull request merge button should be working again. If you are still seeing issues, please attempt refreshing the pull request page.","created_at":"2024-04-24T18:01:13.498Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T18:01:13.498Z","display_at":"2024-04-24T18:01:13.498Z","deliver_notifications":true,"tweet_id":null,"id":"jdjv6xjhd55k","incident_id":"lg4ndssz2n7l","custom_tweet":null,"affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"One of our mitigations from the previous incident caused live updates to the pull request merge button to be disabled for some customers. Refreshing the page will update the mergability status.","created_at":"2024-04-24T17:45:25.329Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T17:45:25.329Z","display_at":"2024-04-24T17:45:25.329Z","deliver_notifications":true,"tweet_id":null,"id":"9p4bgbp2d3x4","incident_id":"lg4ndssz2n7l","custom_tweet":null,"affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Pull Requests","created_at":"2024-04-24T17:40:48.352Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T17:40:48.352Z","display_at":"2024-04-24T17:40:48.352Z","deliver_notifications":true,"tweet_id":null,"id":"n1mzvmy35pbv","incident_id":"lg4ndssz2n7l","custom_tweet":null,"affected_components":[{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Pull Requests","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2024-04-24T18:20:14.311Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":false,"start_date":null,"id":"hhtssxt0f5v2","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Pull Requests, Git Operations, Actions, API Requests, Issues and Webhooks","status":"resolved","created_at":"2024-04-24T10:45:49.074Z","updated_at":"2024-04-30T11:36:11.474Z","monitoring_at":null,"resolved_at":"2024-04-24T16:16:34.627Z","impact":"minor","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/kjyf52x1k2y7","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-24T10:45:49.067Z","reminder_intervals":null,"id":"7txcg0j03kkg","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On April 24, 2024, between 10:26 and 16:30 UTC, the Pull Request page took longer than usual to enable a Pull Request to be merged. During the incident, the 75th percentile time to merge a pull request went from around 3 seconds to just over 6 minutes. We also saw slightly elevated error rates across the service, with an average error rate of 0.1%, peaking to 0.31% of requests. \u003cbr /\u003e\u003cbr /\u003eThe underlying cause of the incident was a repeated problematic query that resulted in MySQL replicas crashing, which caused a back-up in the jobs that compute whether PRs can be merged.\u003cbr /\u003e\u003cbr /\u003eAt 16:30 the incident self-mitigated when the volume of the problematic query decreased. Concurrent to this, we deployed a mitigation to remove the mergability polling code from the PR experience to alleviate pressure on MySQL servers. This mitigation caused the merge button to not automatically enable, but require a page refresh instead. At 17:40 we statused PRs again for this issue. We rolled back the mitigation to resolve the second incident at 18:20.\u003cbr /\u003e\u003cbr /\u003eGoing forward, to improve mitigation we are investing in faster recovery from cascading read-replica crashes, improving query retry logic to more aggressively back off when read replicas are unhealthy, improving our ability to diagnose server crashes and trace them back to user activity, and making it easier to block problematic queries.\u003cbr /\u003e\u003cbr /\u003eTo prevent recurrence, we are eliminating the query that caused the server to crash, improving our detection and mitigation of problematic queries, and investigating and remediating the underlying issues that caused MySQL to crash. We are also improving the merge button polling mechanism and mergeability background job to be resilient to this class of incident so customers can still merge PRs. \u003cbr /\u003e","created_at":"2024-04-24T16:16:34.627Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-30T11:36:11.471Z","display_at":"2024-04-24T16:16:34.627Z","deliver_notifications":true,"tweet_id":null,"id":"zp58z5pkfg6m","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Issues is operating normally.","created_at":"2024-04-24T16:13:44.558Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T16:13:44.558Z","display_at":"2024-04-24T16:13:44.558Z","deliver_notifications":true,"tweet_id":null,"id":"1xmqr8z64709","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Actions is operating normally.","created_at":"2024-04-24T16:13:25.730Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T16:13:25.730Z","display_at":"2024-04-24T16:13:25.730Z","deliver_notifications":true,"tweet_id":null,"id":"qzqfd5jng1ms","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Pull Requests is operating normally.","created_at":"2024-04-24T16:13:25.197Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T16:13:25.197Z","display_at":"2024-04-24T16:13:25.197Z","deliver_notifications":true,"tweet_id":null,"id":"xmrlkhnh2zt3","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Webhooks is operating normally.","created_at":"2024-04-24T16:13:07.743Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T16:13:07.743Z","display_at":"2024-04-24T16:13:07.743Z","deliver_notifications":true,"tweet_id":null,"id":"zp4vbq5w2r23","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Git Operations is operating normally.","created_at":"2024-04-24T16:13:06.533Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T16:13:06.533Z","display_at":"2024-04-24T16:13:06.533Z","deliver_notifications":true,"tweet_id":null,"id":"mxzmk0gftlgv","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"API Requests is operating normally.","created_at":"2024-04-24T16:12:42.166Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T16:12:42.166Z","display_at":"2024-04-24T16:12:42.166Z","deliver_notifications":true,"tweet_id":null,"id":"z5kw4rjmb89t","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are seeing site-wide recovery but continue to closely monitor our systems and putting additional mitigations in place to ensure we are back to full health.","created_at":"2024-04-24T15:50:10.980Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T15:50:10.980Z","display_at":"2024-04-24T15:50:10.980Z","deliver_notifications":true,"tweet_id":null,"id":"3q2hshbh8tp4","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are continuing to see consistent impact, and we’re continuing to work on multiple mitigations to reduce load on our systems.","created_at":"2024-04-24T14:08:27.660Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T14:08:27.660Z","display_at":"2024-04-24T14:08:27.660Z","deliver_notifications":true,"tweet_id":null,"id":"qrz2ky80dmhm","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We have found an issue that may be contributing additional load to the web site and are working on mitigations. We don't see any additional impact at this time and will provide another update within an hour if we see improvements or fully mitigate the issue based on this investigation.","created_at":"2024-04-24T12:47:33.968Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T12:47:33.968Z","display_at":"2024-04-24T12:47:33.968Z","deliver_notifications":true,"tweet_id":null,"id":"3jk2lbml3rl7","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We have taken some mitigations and see less than 0.3 percent of requests failing site wide but we still see elevated 500 errors and will continue to stay statused and investigate until we are confident we have restored our error rate to base line.","created_at":"2024-04-24T12:00:34.266Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T12:00:34.266Z","display_at":"2024-04-24T12:00:34.266Z","deliver_notifications":true,"tweet_id":null,"id":"js90vn0xtxmy","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are seeing increased 500 errors for various GraphQL and REST APIs related to database issues. Some users may see periodic 500 errors. The team is looking into the problematic queries and mitigations now.","created_at":"2024-04-24T11:13:40.330Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T11:13:40.330Z","display_at":"2024-04-24T11:13:40.330Z","deliver_notifications":true,"tweet_id":null,"id":"w8wwj0fdp7cx","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Actions is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-24T11:09:19.213Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T11:09:19.213Z","display_at":"2024-04-24T11:09:19.213Z","deliver_notifications":true,"tweet_id":null,"id":"jb4kcyshs250","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Git Operations is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-24T11:06:52.688Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T11:06:52.688Z","display_at":"2024-04-24T11:06:52.688Z","deliver_notifications":true,"tweet_id":null,"id":"3gxklq1psp8f","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Pull Requests is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-24T10:55:52.334Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T10:55:52.334Z","display_at":"2024-04-24T10:55:52.334Z","deliver_notifications":true,"tweet_id":null,"id":"m0mzx62t2y1j","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"partial_outage","new_status":"partial_outage"},{"code":"hhtssxt0f5v2","name":"Pull Requests","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"Webhooks is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-24T10:52:06.409Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T10:52:06.409Z","display_at":"2024-04-24T10:52:06.409Z","deliver_notifications":true,"tweet_id":null,"id":"1pms9pffzrlh","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"4230lsnqdsld","name":"Webhooks","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-24T10:51:04.443Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T10:51:04.443Z","display_at":"2024-04-24T10:51:04.443Z","deliver_notifications":true,"tweet_id":null,"id":"cynmdnfb176b","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for API Requests","created_at":"2024-04-24T10:45:49.145Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T10:45:49.145Z","display_at":"2024-04-24T10:45:49.145Z","deliver_notifications":true,"tweet_id":null,"id":"392lxszmr401","incident_id":"7txcg0j03kkg","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2024-04-24T16:13:06.507Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2024-04-24T16:12:42.107Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Webhooks","created_at":"2019-11-13T18:00:24.256Z","updated_at":"2024-04-24T16:13:07.705Z","position":3,"description":"Real time HTTP callbacks of user-generated and system events","showcase":false,"start_date":null,"id":"4230lsnqdsld","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2024-04-24T16:13:44.528Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pull Requests","created_at":"2020-09-02T15:39:06.329Z","updated_at":"2024-04-24T18:20:14.311Z","position":6,"description":"Requests for Pull Requests on GitHub.com","showcase":false,"start_date":null,"id":"hhtssxt0f5v2","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2024-04-24T16:13:25.696Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Git Operations","status":"resolved","created_at":"2024-04-24T10:56:01.104Z","updated_at":"2024-04-24T11:01:46.435Z","monitoring_at":null,"resolved_at":"2024-04-24T11:01:46.418Z","impact":"minor","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/kp2nxqjg41ny","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-24T10:56:01.098Z","reminder_intervals":null,"id":"r7641l07b67v","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"This incident has been resolved.","created_at":"2024-04-24T11:01:46.418Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T11:01:46.418Z","display_at":"2024-04-24T11:01:46.418Z","deliver_notifications":true,"tweet_id":null,"id":"8mtl1m1c2bgp","incident_id":"r7641l07b67v","custom_tweet":null,"affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Git Operations","created_at":"2024-04-24T10:56:01.181Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-24T10:56:01.181Z","display_at":"2024-04-24T10:56:01.181Z","deliver_notifications":true,"tweet_id":null,"id":"njdsb9h31pnf","incident_id":"r7641l07b67v","custom_tweet":null,"affected_components":[{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2024-04-24T16:13:06.507Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Codespaces","status":"resolved","created_at":"2024-04-18T18:25:12.880Z","updated_at":"2024-05-01T17:49:09.512Z","monitoring_at":null,"resolved_at":"2024-04-18T18:47:27.083Z","impact":"minor","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/6vg3jh1d12nt","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-18T18:25:12.873Z","reminder_intervals":null,"id":"jnlhc0lml6pp","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Between 4:40 UTC and 18:47 UTC on April 18th, approximately 200 users experienced issues creating and reconnecting to Codespaces. After investigation it was determined that the nearly 100% of those users were internal to GitHub. There was very little customer impact during this incident.\u003cbr /\u003e\u003cbr /\u003eWe are discussing improvements to our monitoring so we can more quickly discern internal and external impact and avoid potentially confusing public status.","created_at":"2024-04-18T18:47:27.083Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-05-01T17:49:09.508Z","display_at":"2024-04-18T18:47:27.083Z","deliver_notifications":true,"tweet_id":null,"id":"16w4b02xzqy4","incident_id":"jnlhc0lml6pp","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Codespaces customers using our 16 core machines in West US 2 and West US 3 region may experience issues creating new Codespaces and resuming existing Codespaces. We suggest any customers experiencing issues switch to the East US region.","created_at":"2024-04-18T18:41:25.213Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-18T18:41:25.213Z","display_at":"2024-04-18T18:41:25.213Z","deliver_notifications":true,"tweet_id":null,"id":"1kx3s5t1rh6v","incident_id":"jnlhc0lml6pp","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Codespaces","created_at":"2024-04-18T18:25:12.949Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-18T18:25:12.949Z","display_at":"2024-04-18T18:25:12.949Z","deliver_notifications":true,"tweet_id":null,"id":"t7kfnzyyfy6b","incident_id":"jnlhc0lml6pp","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Codespaces","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2024-04-18T18:47:26.803Z","position":10,"description":"Orchestration and Compute for GitHub Codespaces","showcase":false,"start_date":"2021-08-11T00:00:00.000Z","id":"h2ftsgbw7kmk","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Copilot","status":"resolved","created_at":"2024-04-16T23:51:05.248Z","updated_at":"2024-04-25T20:49:15.456Z","monitoring_at":null,"resolved_at":"2024-04-17T00:48:52.000Z","impact":"minor","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/g5k8sb9hw9zh","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-16T23:51:05.242Z","reminder_intervals":null,"id":"8jd596s0lt7s","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"On April 16th, 2024, between 22:31 UTC and 00:11 UTC, Copilot chat users experienced elevated request errors. On average, the error rate was 1.2% and peaked at 5.2%. This was due to a rolling application upgrade applied to a backend system during a maintenance event.\u003cbr /\u003e\u003cbr /\u003eThe incident was resolved once the rolling upgrade was completed.\u003cbr /\u003e\u003cbr /\u003eWe are working to improve monitoring and alerting of our services, be more resilient to failures, and coordinate maintenance events to reduce our time to detection and mitigation of issues like this in the future.","created_at":"2024-04-17T00:48:52.096Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-25T20:49:15.437Z","display_at":"2024-04-17T00:48:52.000Z","deliver_notifications":true,"tweet_id":null,"id":"0x713q47kns7","incident_id":"8jd596s0lt7s","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We're continuing to investigate issues with Copilot","created_at":"2024-04-17T00:30:18.660Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-17T00:30:18.660Z","display_at":"2024-04-17T00:30:18.660Z","deliver_notifications":true,"tweet_id":null,"id":"rmhhfg4fdtwm","incident_id":"8jd596s0lt7s","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Copilot is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-16T23:59:09.368Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-16T23:59:09.368Z","display_at":"2024-04-16T23:59:09.368Z","deliver_notifications":true,"tweet_id":null,"id":"5yg0vdcd82ym","incident_id":"8jd596s0lt7s","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"partial_outage"}]},{"status":"investigating","body":"We're investigating issues with Copilot availability","created_at":"2024-04-16T23:57:21.097Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-16T23:57:21.097Z","display_at":"2024-04-16T23:57:21.097Z","deliver_notifications":true,"tweet_id":null,"id":"78lm1wly5f1w","incident_id":"8jd596s0lt7s","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2024-04-16T23:51:05.279Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-16T23:51:05.279Z","display_at":"2024-04-16T23:51:05.279Z","deliver_notifications":true,"tweet_id":null,"id":"390094lghgcp","incident_id":"8jd596s0lt7s","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2024-04-17T00:48:52.072Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Copilot","status":"resolved","created_at":"2024-04-15T12:58:18.723Z","updated_at":"2024-04-25T15:12:08.969Z","monitoring_at":null,"resolved_at":"2024-04-15T14:53:58.208Z","impact":"minor","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/wgnk6b3d9m6s","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-15T12:58:18.718Z","reminder_intervals":null,"id":"clgh4y84qzc8","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Between April 15th, 2024 (09:45 UTC) and April 18th, 2024 (19:10 UTC), Copilot completions experienced intermittent periods of degraded service availability affecting portions of Europe and North America, impacting 3.5% of users globally at its peak. This was due to rolling Operating System level maintenance updates to Copilot infrastructure within those regions, which failed to gracefully restart as intended.\u003cbr /\u003e\u003cbr /\u003eThe incident was mitigated by routing traffic to other regions, and was resolved once the update was completed and normal traffic routing was restored.\u003cbr /\u003e\u003cbr /\u003eWe are working to resolve the root issue that prevented systems from restarting gracefully, as well as improving our coordination and monitoring around backend maintenance operations going forward to reduce time to recovery from such issues in the future.","created_at":"2024-04-15T14:53:58.208Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-25T15:12:08.964Z","display_at":"2024-04-15T14:53:58.208Z","deliver_notifications":true,"tweet_id":null,"id":"kd4dypyd68cw","incident_id":"clgh4y84qzc8","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We have applied mitigation for Copilot in EU region and are working towards the full recovery of the service.","created_at":"2024-04-15T14:13:51.996Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-15T14:13:51.996Z","display_at":"2024-04-15T14:13:51.996Z","deliver_notifications":true,"tweet_id":null,"id":"8t87j05nhvl0","incident_id":"clgh4y84qzc8","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"Due to an outage in one Copilot region traffic is currently being served from other regions. European users may experience higher response times.","created_at":"2024-04-15T13:35:36.603Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-15T13:35:36.603Z","display_at":"2024-04-15T13:35:36.603Z","deliver_notifications":true,"tweet_id":null,"id":"pm7nhybdk40f","incident_id":"clgh4y84qzc8","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Copilot","created_at":"2024-04-15T12:58:18.782Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-15T12:58:18.782Z","display_at":"2024-04-15T12:58:18.782Z","deliver_notifications":true,"tweet_id":null,"id":"6zpqtfgjhg5j","incident_id":"clgh4y84qzc8","custom_tweet":null,"affected_components":[{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2024-04-17T00:48:52.072Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"We are investigating reports of degraded performance.","status":"resolved","created_at":"2024-04-14T14:21:47.663Z","updated_at":"2024-04-17T17:09:33.362Z","monitoring_at":null,"resolved_at":"2024-04-14T21:53:52.824Z","impact":"major","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/xg751bqwtq3h","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-14T14:21:47.655Z","reminder_intervals":null,"id":"b6f4j9ljz9r6","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Beginning at 17:30 UTC on April 11th and lasting until 20:30 UTC on April 14th, github.com saw significant (up to 2 hours) delays in delivering emails. At 14:21 UTC on April 14th, community reports of this were confirmed and an incident declared. Emails most impacted by the delay were password reset and unrecognized device verification, which contain time-sensitive links or verification codes, and are required to be acted on in order for password resets or unrecognized logins to proceed. \u003cbr /\u003e\u003cbr /\u003eUsers attempting to reset their password during the incident were unable to complete the reset. Users without two-factor authentication (2FA), signing in on an unrecognized device, were unable to complete device verification. Enterprise Managed Users, users with 2FA, and users on recognized devices or IP addresses were still able to sign in. This impacted 800-1000 user device verifications and 300-400 password resets. \u003cbr /\u003e\u003cbr /\u003eThe mailer delays were caused by increased usage of a shared resource pool; a separate internal job queue became unhealthy and prevented the mailer queue from being worked on. \u003cbr /\u003e\u003cbr /\u003eWe have made some immediate improvements to better detect and react to this type of situation again. As a short-term mitigation strategy, we have added a queue-bypass ability for  time-sensitive emails, like password reset and unrecognized device verification. We can enable this setting if we observe email delays reoccurring, which will ensure that future incidents do not affect user ability to complete critical login flows. We have paused the unhealthy job queue, to prevent impact to other queues using shared resources. And we have updated our methods of detection for anomalous email delivery, to identify this issue sooner.","created_at":"2024-04-14T21:53:52.824Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-17T17:09:33.359Z","display_at":"2024-04-14T21:53:52.824Z","deliver_notifications":true,"tweet_id":null,"id":"bt3s4pzzfb5v","incident_id":"b6f4j9ljz9r6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are seeing a full recovery. Device verification and password reset emails are delivered on time.","created_at":"2024-04-14T21:52:40.913Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-14T21:52:40.913Z","display_at":"2024-04-14T21:52:40.913Z","deliver_notifications":true,"tweet_id":null,"id":"jxgmp5z428fr","incident_id":"b6f4j9ljz9r6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are deploying a possible mitigation to the delayed device verification and password change emails.","created_at":"2024-04-14T21:34:37.008Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-14T21:34:37.008Z","display_at":"2024-04-14T21:34:37.008Z","deliver_notifications":true,"tweet_id":null,"id":"b5q4w388nf0k","incident_id":"b6f4j9ljz9r6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We continue to investigate issues with delays of email deliveries which is preventing users without 2FA enabled from verifying new devices. We will provide more information as it becomes available.","created_at":"2024-04-14T19:54:15.717Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-14T19:54:15.717Z","display_at":"2024-04-14T19:54:15.717Z","deliver_notifications":true,"tweet_id":null,"id":"w7ct4t09r33k","incident_id":"b6f4j9ljz9r6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are continuing to investigate issues with the delivery of device verification emails for users without 2FA.","created_at":"2024-04-14T15:50:44.290Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-14T15:50:44.290Z","display_at":"2024-04-14T15:50:44.290Z","deliver_notifications":true,"tweet_id":null,"id":"gn6f9qs8101f","incident_id":"b6f4j9ljz9r6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are continuing to investigate issues with the delivery of device verification emails for users without 2FA on new devices.","created_at":"2024-04-14T15:01:35.209Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-14T15:01:35.209Z","display_at":"2024-04-14T15:01:35.209Z","deliver_notifications":true,"tweet_id":null,"id":"vvrcmt5rx1hw","incident_id":"b6f4j9ljz9r6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"Device verification emails for sign-ins for users without 2FA on new devices are being sent late or not at all. This is blocking successful sign-ins for these users. We are investigating.","created_at":"2024-04-14T14:27:17.748Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-14T14:27:17.748Z","display_at":"2024-04-14T14:27:17.748Z","deliver_notifications":true,"tweet_id":null,"id":"qfp4yg2q7dpf","incident_id":"b6f4j9ljz9r6","custom_tweet":null,"affected_components":null},{"status":"investigating","body":"We are currently investigating this issue.","created_at":"2024-04-14T14:21:47.713Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-14T14:21:47.713Z","display_at":"2024-04-14T14:21:47.713Z","deliver_notifications":true,"tweet_id":null,"id":"lbx514bjcvbb","incident_id":"b6f4j9ljz9r6","custom_tweet":null,"affected_components":null}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[]},{"name":"Incident with Git Operations, API Requests, Actions, Pages, Issues and Copilot","status":"resolved","created_at":"2024-04-10T18:41:20.887Z","updated_at":"2024-04-11T19:40:59.475Z","monitoring_at":null,"resolved_at":"2024-04-10T19:03:05.691Z","impact":"major","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/tlpspr2jwz0b","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"major","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-10T18:41:20.880Z","reminder_intervals":null,"id":"9zq8p50p7tkh","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"investigating","body":"On April 10, 2024, between 2024-04-10 18:33 UTC and 2024-04-10 19:03 UTC, several services were degraded due to the release of a compute-intensive database query that prevented a key database cluster from serving other queries. \u003cbr /\u003e\u003cbr /\u003eGitHub Actions saw delays and failures across the entire run life cycle and had a significant increase in the number of  timeouts in API requests. All Pages deployments failed for the duration of the incident. Git Systems saw approximately 12% of raw file download requests and 16% of repository archive download requests return HTTP 50X error codes for the duration of the incident. Issues experienced increased latency for issue creation and updates. Codespaces saw roughly 500 requests to create and resume a Codespace timeout during the incident.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by rolling back the offending query. We are working to introduce measures to automatically detect compute-intensive queries in test runs during CI to prevent an issue like this one from recurring.","created_at":"2024-04-10T19:03:05.797Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-11T19:40:59.465Z","display_at":"2024-04-10T19:03:05.797Z","deliver_notifications":true,"tweet_id":null,"id":"ct0jvn1nd593","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Git Operations, API Requests, Actions, Pages, Issues and Copilot are operating normally.","created_at":"2024-04-10T19:03:05.747Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T19:03:05.747Z","display_at":"2024-04-10T19:03:05.747Z","deliver_notifications":true,"tweet_id":null,"id":"qvfb5wv395fr","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}]},{"status":"resolved","body":"This incident has been resolved.","created_at":"2024-04-10T19:03:05.691Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T19:03:05.691Z","display_at":"2024-04-10T19:03:05.691Z","deliver_notifications":true,"tweet_id":null,"id":"454939tbbgp7","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"operational"}]},{"status":"investigating","body":"Git Operations, API Requests, Actions, Pages, Issues and Copilot are operating normally.","created_at":"2024-04-10T19:03:05.643Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T19:03:05.643Z","display_at":"2024-04-10T19:03:05.643Z","deliver_notifications":true,"tweet_id":null,"id":"kkbqxl2rpk9c","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"operational"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"operational"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"operational"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"operational"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"operational"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"Copilot is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-10T19:01:17.914Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T19:01:17.914Z","display_at":"2024-04-10T19:01:17.914Z","deliver_notifications":true,"tweet_id":null,"id":"3hxb8kkvzg4s","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"major_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We're aware of issues impacting multiple services and have rolled back the deployment. Systems appear to be recovering and we will continue to monitor.","created_at":"2024-04-10T18:55:48.122Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T18:55:48.122Z","display_at":"2024-04-10T18:55:48.122Z","deliver_notifications":true,"tweet_id":null,"id":"0ggn8bf4flgn","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"major_outage","new_status":"major_outage"}]},{"status":"investigating","body":"API Requests is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-10T18:53:20.469Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T18:53:20.469Z","display_at":"2024-04-10T18:53:20.469Z","deliver_notifications":true,"tweet_id":null,"id":"tl69hpfl6zmv","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"major_outage","new_status":"partial_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"major_outage","new_status":"major_outage"}]},{"status":"investigating","body":"Copilot is experiencing degraded availability. We are continuing to investigate.","created_at":"2024-04-10T18:45:21.270Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T18:45:21.270Z","display_at":"2024-04-10T18:45:21.270Z","deliver_notifications":true,"tweet_id":null,"id":"1wy9557sbljk","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"major_outage","new_status":"major_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"partial_outage","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"},{"code":"pjmpxvq2cmr2","name":"Copilot","old_status":"operational","new_status":"major_outage"}]},{"status":"investigating","body":"Issues is experiencing degraded performance. We are continuing to investigate.","created_at":"2024-04-10T18:42:52.208Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T18:42:52.208Z","display_at":"2024-04-10T18:42:52.208Z","deliver_notifications":true,"tweet_id":null,"id":"1znlqvqxwlkb","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"major_outage","new_status":"major_outage"},{"code":"kr09ddfgbfsf","name":"Issues","old_status":"operational","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"API Requests is experiencing degraded availability. We are continuing to investigate.","created_at":"2024-04-10T18:42:15.258Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T18:42:15.258Z","display_at":"2024-04-10T18:42:15.258Z","deliver_notifications":true,"tweet_id":null,"id":"9xbzf65r5rw7","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"partial_outage","new_status":"major_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"partial_outage","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"partial_outage","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Git Operations, API Requests, Actions and Pages","created_at":"2024-04-10T18:41:20.981Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T18:41:20.981Z","display_at":"2024-04-10T18:41:20.981Z","deliver_notifications":true,"tweet_id":null,"id":"pydrcbf4bk4g","incident_id":"9zq8p50p7tkh","custom_tweet":null,"affected_components":[{"code":"brv1bkgrwx7q","name":"API Requests","old_status":"operational","new_status":"partial_outage"},{"code":"vg70hn9s2tyj","name":"Pages","old_status":"operational","new_status":"partial_outage"},{"code":"8l4ygp009s5s","name":"Git Operations","old_status":"operational","new_status":"partial_outage"},{"code":"br0l2tvcx85d","name":"Actions","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Git Operations","created_at":"2017-01-31T20:05:05.370Z","updated_at":"2024-04-24T16:13:06.507Z","position":1,"description":"Performance of git clones, pulls, pushes, and associated operations","showcase":false,"start_date":null,"id":"8l4ygp009s5s","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"API Requests","created_at":"2017-01-31T20:01:46.621Z","updated_at":"2024-04-24T16:12:42.107Z","position":2,"description":"Requests for GitHub APIs","showcase":false,"start_date":null,"id":"brv1bkgrwx7q","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Issues","created_at":"2017-01-31T20:01:46.638Z","updated_at":"2024-04-24T16:13:44.528Z","position":5,"description":"Requests for Issues on GitHub.com","showcase":false,"start_date":null,"id":"kr09ddfgbfsf","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Actions","created_at":"2019-11-13T18:02:19.432Z","updated_at":"2024-04-24T16:13:25.696Z","position":7,"description":"Workflows, Compute and Orchestration for GitHub Actions","showcase":false,"start_date":null,"id":"br0l2tvcx85d","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Pages","created_at":"2017-01-31T20:04:33.923Z","updated_at":"2024-04-10T19:03:05.749Z","position":9,"description":"Frontend application and API servers for Pages builds","showcase":false,"start_date":null,"id":"vg70hn9s2tyj","page_id":"kctbh9vrtdwd","group_id":null},{"status":"operational","name":"Copilot","created_at":"2022-06-21T16:04:33.017Z","updated_at":"2024-04-17T00:48:52.072Z","position":11,"description":null,"showcase":false,"start_date":"2022-06-21T00:00:00.000Z","id":"pjmpxvq2cmr2","page_id":"kctbh9vrtdwd","group_id":null}]},{"name":"Incident with Codespaces","status":"resolved","created_at":"2024-04-10T16:12:30.443Z","updated_at":"2024-04-12T18:28:26.106Z","monitoring_at":null,"resolved_at":"2024-04-10T18:07:51.793Z","impact":"minor","shortlink":"https://web.archive.org/web/20240502121617/https://stspg.io/pppg7ckq30xh","scheduled_for":null,"scheduled_until":null,"scheduled_remind_prior":false,"scheduled_reminded_at":null,"impact_override":"minor","scheduled_auto_in_progress":false,"scheduled_auto_completed":false,"metadata":{},"started_at":"2024-04-10T16:12:30.438Z","reminder_intervals":null,"id":"gqb335zdyq82","page_id":"kctbh9vrtdwd","incident_updates":[{"status":"resolved","body":"Between 2024-04-09 21:35 UTC and 2024-04-10 19:03 UTC, creation of new Codespaces was degraded by an image upgrade to the virtual machines of new Codespaces. During the incident, approximately 7% of new Codespaces were created but never became available to their owning end users.\u003cbr /\u003e\u003cbr /\u003eWe mitigated the incident by reverting to the previous image version. We are working to improve deployment confidence around image upgrades to reduce the likelihood of recurrence.","created_at":"2024-04-10T18:07:51.793Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-12T18:28:26.102Z","display_at":"2024-04-10T18:07:51.793Z","deliver_notifications":true,"tweet_id":null,"id":"cfzf31tc2k94","incident_id":"gqb335zdyq82","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"operational"}]},{"status":"investigating","body":"We have applied a fix and are continuing to monitor. This incident will remain open for now until we have confirmed that the service is fully restored.","created_at":"2024-04-10T17:31:49.691Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T17:31:49.691Z","display_at":"2024-04-10T17:31:49.691Z","deliver_notifications":true,"tweet_id":null,"id":"6hl2tqf5x85r","incident_id":"gqb335zdyq82","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We believe we have identified the root cause of the issue and are working to fully restore the Codespaces service. We will provide another update within the next 30 minutes.","created_at":"2024-04-10T16:56:21.697Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T16:56:21.697Z","display_at":"2024-04-10T16:56:21.697Z","deliver_notifications":true,"tweet_id":null,"id":"djkfmjh0g373","incident_id":"gqb335zdyq82","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We’re seeing issues related to connecting to Codespaces impacting a subset of users. We are actively investigating and will provide another update shortly.","created_at":"2024-04-10T16:20:04.411Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T16:20:04.411Z","display_at":"2024-04-10T16:20:04.411Z","deliver_notifications":true,"tweet_id":null,"id":"pkmr2qcptft5","incident_id":"gqb335zdyq82","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"partial_outage","new_status":"partial_outage"}]},{"status":"investigating","body":"We are investigating reports of degraded performance for Codespaces","created_at":"2024-04-10T16:12:30.506Z","wants_twitter_update":false,"twitter_updated_at":null,"updated_at":"2024-04-10T16:12:30.506Z","display_at":"2024-04-10T16:12:30.506Z","deliver_notifications":true,"tweet_id":null,"id":"l1kfb1qp1xdn","incident_id":"gqb335zdyq82","custom_tweet":null,"affected_components":[{"code":"h2ftsgbw7kmk","name":"Codespaces","old_status":"operational","new_status":"partial_outage"}]}],"postmortem_body":null,"postmortem_body_last_updated_at":null,"postmortem_ignored":false,"postmortem_published_at":null,"postmortem_notified_subscribers":false,"postmortem_notified_twitter":false,"components":[{"status":"operational","name":"Codespaces","created_at":"2021-08-11T16:02:09.505Z","updated_at":"2024-04-18T18:47:26.803Z","position":10,"description":"Orchestration and Compute for GitHub Codespaces","showcase":false,"start_date":"2021-08-11T00:00:00.000Z","id":"h2ftsgbw7kmk","page_id":"kctbh9vrtdwd","group_id":null}]}]
}